Merge branch 'master' of gnunet.org:gnunet

361 files changed, 20859 insertions, 44381 deletions

diff --git a/src/arm/arm_api.c b/src/arm/arm_api.c
index a61343833..56af544b1 100644
--- a/src/arm/arm_api.c
+++ b/src/arm/arm_api.c
@@ -753,7 +753,7 @@ change_service (struct GNUNET_ARM_Handle *h,
 
   slen = strlen (service_name) + 1;
   if (slen + sizeof (struct GNUNET_ARM_Message) >=
-      GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/arm/gnunet-arm.c b/src/arm/gnunet-arm.c
index 65a6463cf..78c78738a 100644
--- a/src/arm/gnunet-arm.c
+++ b/src/arm/gnunet-arm.c
@@ -121,12 +121,12 @@ static struct GNUNET_SCHEDULER_Task *timeout_task;
 /**
  * Do we want to give our stdout to gnunet-service-arm?
  */
-static unsigned int no_stdout;
+static int no_stdout;
 
 /**
  * Do we want to give our stderr to gnunet-service-arm?
  */
-static unsigned int no_stderr;
+static int no_stderr;
 
 /**
  * Handle for the task running the #action_loop().
@@ -220,14 +220,8 @@ req_string (enum GNUNET_ARM_RequestStatus rs)
   {
   case GNUNET_ARM_REQUEST_SENT_OK:
     return _("Message was sent successfully");
-  case GNUNET_ARM_REQUEST_CONFIGURATION_ERROR:
-    return _("Misconfiguration (can not connect to the ARM service)");
   case GNUNET_ARM_REQUEST_DISCONNECTED:
     return _("We disconnected from ARM before we could send a request");
-  case GNUNET_ARM_REQUEST_BUSY:
-    return _("ARM API is busy");
-  case GNUNET_ARM_REQUEST_TIMEOUT:
-    return _("Request timed out");
   }
   return _("Unknown request status");
 }
@@ -245,27 +239,27 @@ ret_string (enum GNUNET_ARM_Result result)
   switch (result)
   {
   case GNUNET_ARM_RESULT_STOPPED:
-    return _("%s is stopped");
+    return _("is stopped");
   case GNUNET_ARM_RESULT_STARTING:
-    return _("%s is starting");
+    return _("is starting");
   case GNUNET_ARM_RESULT_STOPPING:
-    return _("%s is stopping");
+    return _("is stopping");
   case GNUNET_ARM_RESULT_IS_STARTING_ALREADY:
-    return _("%s is starting already");
+    return _("is starting already");
   case GNUNET_ARM_RESULT_IS_STOPPING_ALREADY:
-    return _("%s is stopping already");
+    return _("is stopping already");
   case GNUNET_ARM_RESULT_IS_STARTED_ALREADY:
-    return _("%s is started already");
+    return _("is started already");
   case GNUNET_ARM_RESULT_IS_STOPPED_ALREADY:
-    return _("%s is stopped already");
+    return _("is stopped already");
   case GNUNET_ARM_RESULT_IS_NOT_KNOWN:
-    return _("%s service is not known to ARM");
+    return _("service is not known to ARM");
   case GNUNET_ARM_RESULT_START_FAILED:
-    return _("%s service failed to start");
+    return _("service failed to start");
   case GNUNET_ARM_RESULT_IN_SHUTDOWN:
-    return _("%s service cannot be started because ARM is shutting down");
+    return _("service cannot be manipulated because ARM is shutting down");
   }
-  return _("%.s Unknown result code.");
+  return _("Unknown result code.");
 }
 
 
@@ -378,10 +372,9 @@ stop_callback (void *cls,
       (GNUNET_ARM_RESULT_STOPPED != result) &&
       (GNUNET_ARM_RESULT_IS_STOPPED_ALREADY != result))
   {
-    GNUNET_asprintf (&msg, "%s",
-                     _("Failed to stop the ARM service: %s\n"));
-    FPRINTF (stdout, msg, ret_string (result));
-    GNUNET_free (msg);
+    FPRINTF (stdout,
+             _("Failed to stop the ARM service: %s\n"),
+             ret_string (result));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
@@ -476,11 +469,10 @@ term_callback (void *cls,
   if ((GNUNET_ARM_RESULT_STOPPED != result) &&
       (GNUNET_ARM_RESULT_IS_STOPPED_ALREADY != result))
   {
-    GNUNET_asprintf (&msg,
-                     _("Failed to kill the `%s' service: %s\n"),
-                     term, ret_string (result));
-    FPRINTF (stdout, "%s", msg);
-    GNUNET_free (msg);
+    FPRINTF (stdout,
+             _("Failed to kill the `%s' service: %s\n"),
+             term,
+             ret_string (result));
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
@@ -770,35 +762,70 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'e', "end", NULL, gettext_noop ("stop all GNUnet services"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &end},
-    {'i', "init", "SERVICE", gettext_noop ("start a particular service"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &init},
-    {'k', "kill", "SERVICE", gettext_noop ("stop a particular service"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &term},
-    {'s', "start", NULL, gettext_noop ("start all GNUnet default services"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &start},
-    {'r', "restart", NULL,
-     gettext_noop ("stop and start all GNUnet default services"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &restart},
-    {'d', "delete", NULL,
-     gettext_noop ("delete config file and directory on exit"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &delete},
-    {'m', "monitor", NULL,
-     gettext_noop ("monitor ARM activities"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &monitor},
-    {'q', "quiet", NULL, gettext_noop ("don't print status messages"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &quiet},
-    {'T', "timeout", "DELAY",
-     gettext_noop ("exit with error status if operation does not finish after DELAY"),
-     GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &timeout},
-    {'I', "info", NULL, gettext_noop ("list currently running services"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &list},
-    {'O', "no-stdout", NULL, gettext_noop ("don't let gnunet-service-arm inherit standard output"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &no_stdout},
-    {'E', "no-stderr", NULL, gettext_noop ("don't let gnunet-service-arm inherit standard error"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &no_stderr},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('e',
+                                  "end",
+                                  gettext_noop ("stop all GNUnet services"),
+                                  &end),
+
+    GNUNET_GETOPT_OPTION_STRING ('i',
+                                 "init",
+                                 "SERVICE",
+                                 gettext_noop ("start a particular service"),
+                                 &init),
+
+    GNUNET_GETOPT_OPTION_STRING ('k',
+                                 "kill",
+                                 "SERVICE",
+                                 gettext_noop ("stop a particular service"),
+                                 &term),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('s',
+                                  "start",
+                                  gettext_noop ("start all GNUnet default services"),
+                                  &start),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('r',
+                                  "restart",
+                                  gettext_noop ("stop and start all GNUnet default services"),
+                                  &restart),
+    GNUNET_GETOPT_OPTION_SET_ONE ('d',
+                                  "delete",
+                                  gettext_noop ("delete config file and directory on exit"),
+                                  &delete),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('m',
+                                  "monitor",
+                                  gettext_noop ("monitor ARM activities"),
+                                  &monitor),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('q',
+                                  "quiet",
+                                  gettext_noop ("don't print status messages"),
+                                  &quiet),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('T',
+                                            "timeout",
+                                            "DELAY",
+                                            gettext_noop ("exit with error status if operation does not finish after DELAY"),
+                                            &timeout),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('I',
+                                  "info",
+                                  gettext_noop ("list currently running services"),
+                                  &list), 
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('O',
+                                  "no-stdout",
+                                  gettext_noop ("don't let gnunet-service-arm inherit standard output"),
+                                  &no_stdout),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('E',
+                                  "no-stderr",
+                                  gettext_noop ("don't let gnunet-service-arm inherit standard error"),
+                                  &no_stderr),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/arm/gnunet-service-arm.c b/src/arm/gnunet-service-arm.c
index 4f3e964e3..19088c5cb 100644
--- a/src/arm/gnunet-service-arm.c
+++ b/src/arm/gnunet-service-arm.c
@@ -812,6 +812,7 @@ start_process (struct ServiceList *sl,
                        "%s %s",
                        fin_options,
                        optpos);
+      GNUNET_free (fin_options);
       GNUNET_free (optpos);
     }
     else
@@ -2224,7 +2225,7 @@ main (int argc,
   shc_chld =
     GNUNET_SIGNAL_handler_install (GNUNET_SIGCHLD,
                                    &sighandler_child_death);
-  ret = GNUNET_SERVICE_ruN_ (argc,
+  ret = GNUNET_SERVICE_run_ (argc,
                              argv,
                              "arm",
                              GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN,
diff --git a/src/ats-tests/gnunet-ats-sim.c b/src/ats-tests/gnunet-ats-sim.c
index 56f8f2223..0f32df511 100644
--- a/src/ats-tests/gnunet-ats-sim.c
+++ b/src/ats-tests/gnunet-ats-sim.c
@@ -81,15 +81,19 @@ evaluate (struct GNUNET_TIME_Relative duration_total)
 
 
   duration = (duration_total.rel_value_us / (1000 * 1000));
+  if (0 == duration)
+    duration = 1;
   for (c_m = 0; c_m < e->num_masters; c_m++)
   {
     mp = &masters_p[c_m];
     fprintf (stderr,
-        _("Master [%u]: sent: %u KiB in %u sec. = %u KiB/s, received: %u KiB in %u sec. = %u KiB/s\n"),
-        mp->no, mp->total_bytes_sent / 1024, duration,
-        (mp->total_bytes_sent / 1024) / duration,
-        mp->total_bytes_received / 1024, duration,
-        (mp->total_bytes_received / 1024) / duration);
+             _("Master [%u]: sent: %u KiB in %u sec. = %u KiB/s, received: %u KiB in %u sec. = %u KiB/s\n"),
+             mp->no, mp->total_bytes_sent / 1024,
+             duration,
+             (mp->total_bytes_sent / 1024) / duration,
+             mp->total_bytes_received / 1024,
+             duration,
+             (mp->total_bytes_received / 1024) / duration);
 
     for (c_s = 0; c_s < e->num_slaves; c_s++)
     {
diff --git a/src/ats-tests/gnunet-solver-eval.c b/src/ats-tests/gnunet-solver-eval.c
index 0e49a3a32..1bb7fdee7 100644
--- a/src/ats-tests/gnunet-solver-eval.c
+++ b/src/ats-tests/gnunet-solver-eval.c
@@ -331,12 +331,14 @@ load_episode (struct Experiment *e,
             o->pref_type = GNUNET_ATS_PREFERENCE_LATENCY;
           else
           {
-              fprintf (stderr, "Invalid preference in operation %u `%s' in episode %u\n",
-                  op_counter, op, cur->id);
+              fprintf (stderr,
+                       "Invalid preference in operation %u `%s' in episode %u\n",
+                       op_counter,
+                       op,
+                       cur->id);
               GNUNET_free (type);
               GNUNET_free (op_name);
               GNUNET_free (op);
-              GNUNET_free (pref);
               GNUNET_free (sec_name);
               GNUNET_free_non_null (pref);
               GNUNET_free (o);
@@ -929,22 +931,35 @@ main (int argc, char *argv[])
   opt_log = GNUNET_NO;
   opt_plot = GNUNET_NO;
 
-  static struct GNUNET_GETOPT_CommandLineOption options[] =
-  {
-    { 's', "solver", NULL,
-        gettext_noop ("solver to use"),
-        1, &GNUNET_GETOPT_set_string, &opt_solver},
-    {  'e', "experiment", NULL,
-      gettext_noop ("experiment to use"),
-      1, &GNUNET_GETOPT_set_string, &opt_exp_file},
-    {  'e', "experiment", NULL,
-      gettext_noop ("experiment to use"),
-      1, &GNUNET_GETOPT_set_one, &opt_verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] =
+  {
+    GNUNET_GETOPT_OPTION_STRING ('s',
+                                 "solver",
+                                 NULL,
+                                 gettext_noop ("solver to use"),
+                                 &opt_solver),
+
+    GNUNET_GETOPT_OPTION_STRING ('e',
+                                 "experiment",
+                                 NULL,
+                                 gettext_noop ("experiment to use"),
+                                 &opt_exp_file),
+ 
+    GNUNET_GETOPT_OPTION_SET_ONE ('e',
+                                  "experiment",
+                                  gettext_noop ("experiment to use"),
+                                  &opt_verbose),
     GNUNET_GETOPT_OPTION_END
   };
 
-  GNUNET_PROGRAM_run (argc, argv, argv[0], NULL, options, &run, argv[0]);
+  if (GNUNET_OK !=
+      GNUNET_PROGRAM_run (argc,
+                          argv, argv[0],
+                          NULL,
+                          options,
+                          &run, argv[0]))
+    return 1;
 
   return 0;
 }
-/* end of file ats-testing-experiment.c*/
+/* end of file gnunet-solver-eval.c*/
diff --git a/src/ats-tool/gnunet-ats.c b/src/ats-tool/gnunet-ats.c
index 5fc1d6e92..f645ba56d 100644
--- a/src/ats-tool/gnunet-ats.c
+++ b/src/ats-tool/gnunet-ats.c
@@ -944,34 +944,62 @@ main (int argc,
   stat_receive_done = GNUNET_NO;
   opt_type_str = NULL;
 
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'u', "used", NULL,
-      gettext_noop ("get list of active addresses currently used"), 0,
-      &GNUNET_GETOPT_set_one, &opt_list_used },
-    { 'a', "all", NULL, gettext_noop ("get list of all active addresses"), 0,
-      &GNUNET_GETOPT_set_one, &opt_list_all },
-    { 'C', "connect", "PEER",
-      gettext_noop ("connect to PEER"), 1,
-      &GNUNET_GETOPT_set_string, &cpid_str },
-    { 'n', "numeric", NULL,
-      gettext_noop ("do not resolve IP addresses to hostnames"), 0,
-      &GNUNET_GETOPT_set_one, &opt_resolve_addresses_numeric },
-    { 'm', "monitor", NULL, gettext_noop ("monitor mode"), 0,
-      &GNUNET_GETOPT_set_one, &opt_monitor },
-    { 'p', "preference", NULL, gettext_noop ("set preference for the given peer"),
-      0, &GNUNET_GETOPT_set_one, &opt_set_pref },
-    { 'q', "quotas", NULL, gettext_noop ("print all configured quotas"), 0,
-      &GNUNET_GETOPT_set_one, &opt_print_quotas },
-    { 'i', "id", "TYPE", gettext_noop ("peer id"), 1, &GNUNET_GETOPT_set_string,
-      &opt_pid_str },
-    { 't', "type", "TYPE",
-      gettext_noop ("preference type to set: latency | bandwidth"), 1,
-      &GNUNET_GETOPT_set_string, &opt_type_str },
-    { 'k', "value", "VALUE", gettext_noop ("preference value"), 1,
-      &GNUNET_GETOPT_set_uint, &opt_pref_value },
-    { 'V', "verbose", NULL,
-      gettext_noop ("verbose output (include ATS address properties)"), 0,
-      &GNUNET_GETOPT_set_one, &opt_verbose },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('u',
+                                  "used",
+                                  gettext_noop ("get list of active addresses currently used"),
+                                  &opt_list_used),
+    GNUNET_GETOPT_OPTION_SET_ONE ('a',
+                                  "all",
+                                  gettext_noop ("get list of all active addresses"),
+                                  &opt_list_all),
+
+    GNUNET_GETOPT_OPTION_STRING ('C',
+                                 "connect",
+                                 NULL,
+                                 gettext_noop ("connect to PEER"),
+                                 &cpid_str),
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "numeric",
+                                  gettext_noop ("do not resolve IP addresses to hostnames"),
+                                  &opt_resolve_addresses_numeric),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('m',
+                                  "monitor",
+                                  gettext_noop ("monitor mode"),
+                                  &opt_monitor),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('p',
+                                  "preference",
+                                  gettext_noop ("set preference for the given peer"),
+                                  &opt_set_pref),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('q', 
+                                  "quotas",
+                                  gettext_noop ("print all configured quotas"),
+                                  &opt_print_quotas),
+    GNUNET_GETOPT_OPTION_STRING ('i',
+                                 "id",
+                                 "TYPE",
+                                 gettext_noop ("peer id"),
+                                 &opt_pid_str),
+
+    GNUNET_GETOPT_OPTION_STRING ('t',
+                                 "type",
+                                 "TYPE",
+                                 gettext_noop ("preference type to set: latency | bandwidth"),
+                                 &opt_type_str),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('k',
+                                   "value",
+                                   "VALUE",
+                                   gettext_noop ("preference value"),
+                                   &opt_pref_value),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('V',
+                                  "verbose",
+                                  gettext_noop ("verbose output (include ATS address properties)"),
+                                  &opt_verbose),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/ats/ats_api_scheduling.c b/src/ats/ats_api_scheduling.c
index faeeb6081..81ae01b6a 100644
--- a/src/ats/ats_api_scheduling.c
+++ b/src/ats/ats_api_scheduling.c
@@ -657,9 +657,9 @@ GNUNET_ATS_address_add (struct GNUNET_ATS_SchedulingHandle *sh,
   GNUNET_break (GNUNET_ATS_NET_UNSPECIFIED != prop->scope);
   namelen = strlen (address->transport_name) + 1;
   msize = address->address_length + namelen;
-  if ((msize + sizeof (struct AddressUpdateMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-      (address->address_length >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-      (namelen >= GNUNET_SERVER_MAX_MESSAGE_SIZE) )
+  if ((msize + sizeof (struct AddressUpdateMessage) >= GNUNET_MAX_MESSAGE_SIZE) ||
+      (address->address_length >= GNUNET_MAX_MESSAGE_SIZE) ||
+      (namelen >= GNUNET_MAX_MESSAGE_SIZE) )
   {
     /* address too large for us, this should not happen */
     GNUNET_break (0);
diff --git a/src/ats/gnunet-ats-solver-eval.c b/src/ats/gnunet-ats-solver-eval.c
index e2e68562c..597285f5e 100644
--- a/src/ats/gnunet-ats-solver-eval.c
+++ b/src/ats/gnunet-ats-solver-eval.c
@@ -3289,24 +3289,33 @@ main (int argc, char *argv[])
 
   static struct GNUNET_GETOPT_CommandLineOption options[] =
   {
-    { 's', "solver", NULL,
-        gettext_noop ("solver to use"),
-        1, &GNUNET_GETOPT_set_string, &opt_solver},
-    {  'e', "experiment", NULL,
-      gettext_noop ("experiment to use"),
-      1, &GNUNET_GETOPT_set_string, &opt_exp_file},
-    {  'V', "verbose", NULL,
-      gettext_noop ("be verbose"),
-      0, &GNUNET_GETOPT_set_one, &opt_verbose},
-    {  'p', "print", NULL,
-      gettext_noop ("print logging"),
-      0, &GNUNET_GETOPT_set_one, &opt_print},
-    {  'f', "file", NULL,
-        gettext_noop ("save logging to disk"),
-        0, &GNUNET_GETOPT_set_one, &opt_save},
-    {  'd', "dn", NULL,
-        gettext_noop ("disable normalization"),
-        0, &GNUNET_GETOPT_set_one, &opt_disable_normalization},
+    GNUNET_GETOPT_OPTION_STRING ('s',
+                                 "solver",
+                                 gettext_noop ("solver to use"),
+                                 &opt_solver),
+
+    GNUNET_GETOPT_OPTION_STRING ('e',
+                                 "experiment"
+                                 gettext_noop ("experiment to use"),
+                                 &opt_exp_file),
+
+    GNUNET_GETOPT_OPTION_VERBOSE (&opt_verbose),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('p', 
+                                  "print",
+                                  gettext_noop ("print logging"),
+                                  &opt_print),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('f',
+                                  "file",
+                                  gettext_noop ("save logging to disk"),
+                                  &opt_save),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('d', 
+                                  "dn",
+                                  gettext_noop ("disable normalization"),
+                                  &opt_disable_normalization),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/ats/gnunet-service-ats_addresses.c b/src/ats/gnunet-service-ats_addresses.c
index 1a4a33206..ba34cbacb 100644
--- a/src/ats/gnunet-service-ats_addresses.c
+++ b/src/ats/gnunet-service-ats_addresses.c
@@ -571,7 +571,7 @@ transmit_req_addr (struct AddressIteration *ai,
   msize = plugin_addr_len + plugin_name_length;
 
   GNUNET_assert (sizeof (struct PeerInformationMessage) + msize
-                 < GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                 < GNUNET_MAX_MESSAGE_SIZE);
   env = GNUNET_MQ_msg_extra (msg,
                              msize,
                              GNUNET_MESSAGE_TYPE_ATS_ADDRESSLIST_RESPONSE);
diff --git a/src/ats/gnunet-service-ats_performance.c b/src/ats/gnunet-service-ats_performance.c
index 5252a71bc..07ddf0531 100644
--- a/src/ats/gnunet-service-ats_performance.c
+++ b/src/ats/gnunet-service-ats_performance.c
@@ -85,7 +85,7 @@ notify_client (struct GNUNET_SERVICE_Client *client,
 
   if (NULL != prop)
     GNUNET_break (GNUNET_ATS_NET_UNSPECIFIED != prop->scope);
-  GNUNET_assert (msize < GNUNET_SERVER_MAX_MESSAGE_SIZE);
+  GNUNET_assert (msize < GNUNET_MAX_MESSAGE_SIZE);
   msg = (struct PeerInformationMessage *) buf;
   msg->header.size = htons (msize);
   msg->header.type = htons (GNUNET_MESSAGE_TYPE_ATS_PEER_INFORMATION);
diff --git a/src/ats/perf_ats_solver.c b/src/ats/perf_ats_solver.c
index f05668e9b..855899b36 100644
--- a/src/ats/perf_ats_solver.c
+++ b/src/ats/perf_ats_solver.c
@@ -1404,27 +1404,42 @@ main (int argc, char *argv[])
   ph.total_iterations = 1;
 
   static struct GNUNET_GETOPT_CommandLineOption options[] = {
-      { 'a', "addresses", NULL,
-          gettext_noop ("addresses to use"),
-          1, &GNUNET_GETOPT_set_uint, &ph.N_address },
-      { 's', "start", NULL,
-          gettext_noop ("start with peer"),
-          1, &GNUNET_GETOPT_set_uint, &ph.N_peers_start },
-      { 'e', "end", NULL,
-          gettext_noop ("end with peer"),
-          1, &GNUNET_GETOPT_set_uint, &ph.N_peers_end },
-      { 'i', "iterations", NULL,
-          gettext_noop ("number of iterations used for averaging (default: 1)"),
-          1, &GNUNET_GETOPT_set_uint, &ph.total_iterations },
-      { 'p', "percentage", NULL,
-          gettext_noop ("update a fix percentage of addresses"),
-          1, &GNUNET_GETOPT_set_uint, &ph.opt_update_percent },
-      { 'd', "data", NULL,
-          gettext_noop ("create data file"),
-          0, &GNUNET_GETOPT_set_one, &ph.create_datafile},
-      { 'u', "update", NULL,
-          gettext_noop ("measure updates"),
-          0, &GNUNET_GETOPT_set_one, &ph.measure_updates},
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('a',
+                                     "addresses",
+                                     gettext_noop ("addresses to use"),
+                                     &ph.N_address),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('s',
+                                     "start",
+                                     gettext_noop ("start with peer"),
+                                     &ph.N_peers_start),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('e',
+                                     "end",
+                                     gettext_noop ("end with peer"),
+                                     &ph.N_peers_end),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('i',
+                                     "iterations",
+                                     gettext_noop ("number of iterations used for averaging (default: 1)"),
+                                     &ph.total_iterations),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('p',
+                                     "percentage",
+                                     gettext_noop ("update a fix percentage of addresses"),
+                                     &ph.opt_update_percent),
+
+      GNUNET_GETOPT_OPTION_SET_ONE ('d',
+                                    "data",
+                                    gettext_noop ("create data file"),
+                                    &ph.create_datafile),
+
+      GNUNET_GETOPT_OPTION_SET_ONE ('u',
+                                    "update",
+                                    gettext_noop ("measure updates"),
+                                    &ph.measure_updates),
+
       GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/auction/gnunet-auction-create.c b/src/auction/gnunet-auction-create.c
index a4c029572..e6fcab097 100644
--- a/src/auction/gnunet-auction-create.c
+++ b/src/auction/gnunet-auction-create.c
@@ -155,30 +155,49 @@ fail:
 int
 main (int argc, char *const *argv)
 {
-        static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-                {'d', "description", "FILE",
-                        gettext_noop ("description of the item to be sold"),
-                        1, &GNUNET_GETOPT_set_filename, &fndesc},
-                {'p', "pricemap", "FILE",
-                        gettext_noop ("mapping of possible prices"),
-                        1, &GNUNET_GETOPT_set_filename, &fnprices},
-                {'r', "roundtime", "DURATION",
-                        gettext_noop ("max duration per round"),
-                        1, &GNUNET_GETOPT_set_relative_time, &dround},
-                {'s', "regtime", "DURATION",
-                        gettext_noop ("duration until auction starts"),
-                        1, &GNUNET_GETOPT_set_relative_time, &dstart},
-                {'m', "m", "NUMBER",
-                        gettext_noop ("number of items to sell\n"
-                                      "0 for first price auction\n"
-                                      ">0 for vickrey/M+1st price auction"),
-                        1, &GNUNET_GETOPT_set_uint, &m},
-                {'u', "public", NULL,
-                        gettext_noop ("public auction outcome"),
-                        0, &GNUNET_GETOPT_set_one, &outcome},
-                {'i', "interactive", NULL,
-                        gettext_noop ("keep running in foreground until auction completes"),
-                        0, &GNUNET_GETOPT_set_one, &interactive},
+        struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+                GNUNET_GETOPT_OPTION_FILENAME ('d',
+                                               "description",
+                                               "FILE",
+                                               gettext_noop ("description of the item to be sold"),
+                                               &fndesc),
+
+                GNUNET_GETOPT_OPTION_FILENAME ('p',
+                                               "pricemap",
+                                               "FILE",
+                                               gettext_noop ("mapping of possible prices"),
+                                               &fnprices),
+
+                GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('r',
+                                                        "roundtime",
+                                                        "DURATION",
+                                                        gettext_noop ("max duration per round"),
+                                                        &dround),
+
+                GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('s',
+                                                        "regtime",
+                                                        "DURATION",
+                                                        gettext_noop ("duration until auction starts"),
+                                                        &dstart),
+                GNUNET_GETOPT_OPTION_SET_UINT ('m',
+                                               "m",
+                                               "NUMBER",
+                                               gettext_noop ("number of items to sell\n"
+                                                             "0 for first price auction\n"
+                                                             ">0 for vickrey/M+1st price auction"),
+                                               &m), 
+
+                GNUNET_GETOPT_OPTION_SET_ONE ('u',
+                                              "public",
+                                              gettext_noop ("public auction outcome"),
+                                              &outcome),
+
+                GNUNET_GETOPT_OPTION_SET_ONE ('i',
+                                              "interactive",
+                                              gettext_noop ("keep running in foreground until auction completes"),
+                                              &interactive),
+
                 GNUNET_GETOPT_OPTION_END
         };
         if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/block/bg_bf.c b/src/block/bg_bf.c
index 1a17ec84e..3e7d38892 100644
--- a/src/block/bg_bf.c
+++ b/src/block/bg_bf.c
@@ -77,6 +77,7 @@ bf_group_serialize_cb (struct GNUNET_BLOCK_Group *bg,
                                                  raw,
                                                  gi->bf_size))
   {
+    GNUNET_free (raw);
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
diff --git a/src/cadet/Makefile.am b/src/cadet/Makefile.am
index b52079b2e..ce30ebe46 100644
--- a/src/cadet/Makefile.am
+++ b/src/cadet/Makefile.am
@@ -23,84 +23,48 @@ AM_CLFAGS = -g
 
 libexec_PROGRAMS = \
  gnunet-service-cadet \
- gnunet-service-cadet-new \
  $(EXP_LIBEXEC)
 
 bin_PROGRAMS = \
  gnunet-cadet
 
 lib_LTLIBRARIES = \
-  libgnunetcadetnew.la \
   libgnunetcadet.la \
   $(EXP_LIB)
 
 libgnunetcadet_la_SOURCES = \
-  cadet_api.c cadet_common.c
+  cadet_api.c
 libgnunetcadet_la_LIBADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(XLIB) \
   $(LTLIBINTL)
 libgnunetcadet_la_LDFLAGS = \
   $(GN_LIB_LDFLAGS) $(WINFLAGS) \
-  -version-info 5:0:0
-
-
-libgnunetcadetnew_la_SOURCES = \
-  cadet_api_new.c
-libgnunetcadetnew_la_LIBADD = \
-  $(top_builddir)/src/util/libgnunetutil.la \
-  $(XLIB) \
-  $(LTLIBINTL)
-libgnunetcadetnew_la_LDFLAGS = \
-  $(GN_LIB_LDFLAGS) $(WINFLAGS) \
-  -version-info 6:0:0
+  -version-info 7:0:0
 
 gnunet_cadet_SOURCES = \
   gnunet-cadet.c
 gnunet_cadet_LDADD = \
-  libgnunetcadetnew.la \
+  libgnunetcadet.la \
   $(top_builddir)/src/util/libgnunetutil.la
 
-gnunet_service_cadet_new_SOURCES = \
- gnunet-service-cadet-new.c gnunet-service-cadet-new.h \
- gnunet-service-cadet-new_channel.c gnunet-service-cadet-new_channel.h \
- gnunet-service-cadet-new_connection.c gnunet-service-cadet-new_connection.h \
- gnunet-service-cadet-new_core.c gnunet-service-cadet-new_core.h \
- gnunet-service-cadet-new_dht.c gnunet-service-cadet-new_dht.h \
- gnunet-service-cadet-new_hello.c gnunet-service-cadet-new_hello.h \
- gnunet-service-cadet-new_tunnels.c gnunet-service-cadet-new_tunnels.h \
- gnunet-service-cadet-new_paths.c gnunet-service-cadet-new_paths.h \
- gnunet-service-cadet-new_peer.c gnunet-service-cadet-new_peer.h
-gnunet_service_cadet_new_LDADD = \
-  $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/ats/libgnunetats.la \
-  $(top_builddir)/src/core/libgnunetcore.la \
-  $(top_builddir)/src/dht/libgnunetdht.la \
-  $(top_builddir)/src/statistics/libgnunetstatistics.la \
-  $(top_builddir)/src/transport/libgnunettransport.la \
-  $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
-  $(top_builddir)/src/hello/libgnunethello.la \
-  $(top_builddir)/src/block/libgnunetblock.la
-
 gnunet_service_cadet_SOURCES = \
- gnunet-service-cadet_tunnel.c gnunet-service-cadet_tunnel.h \
- gnunet-service-cadet_connection.c gnunet-service-cadet_connection.h \
+ gnunet-service-cadet.c gnunet-service-cadet.h \
  gnunet-service-cadet_channel.c gnunet-service-cadet_channel.h \
- gnunet-service-cadet_local.c gnunet-service-cadet_local.h \
- gnunet-service-cadet_peer.c gnunet-service-cadet_peer.h \
+ gnunet-service-cadet_connection.c gnunet-service-cadet_connection.h \
+ gnunet-service-cadet_core.c gnunet-service-cadet_core.h \
  gnunet-service-cadet_dht.c gnunet-service-cadet_dht.h \
  gnunet-service-cadet_hello.c gnunet-service-cadet_hello.h \
- cadet_path.c cadet_path.h \
- cadet_common.c \
- gnunet-service-cadet.c
-gnunet_service_cadet_CFLAGS = $(AM_CFLAGS)
+ gnunet-service-cadet_tunnels.c gnunet-service-cadet_tunnels.h \
+ gnunet-service-cadet_paths.c gnunet-service-cadet_paths.h \
+ gnunet-service-cadet_peer.c gnunet-service-cadet_peer.h
 gnunet_service_cadet_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/transport/libgnunettransport.la \
-  $(top_builddir)/src/core/libgnunetcore.la \
   $(top_builddir)/src/ats/libgnunetats.la \
+  $(top_builddir)/src/core/libgnunetcore.la \
   $(top_builddir)/src/dht/libgnunetdht.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
+  $(top_builddir)/src/transport/libgnunettransport.la \
   $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
   $(top_builddir)/src/hello/libgnunethello.la \
   $(top_builddir)/src/block/libgnunetblock.la
@@ -110,39 +74,14 @@ endif
 
 
 if HAVE_TESTING
- noinst_LTLIBRARIES = libgnunetcadettest.la libgnunetcadettestnew.la $(noinst_LIB_EXP)
- noinst_PROGRAMS = gnunet-cadet-profiler
+ noinst_LTLIBRARIES = libgnunetcadettest.la $(noinst_LIB_EXP)
+# noinst_PROGRAMS = gnunet-cadet-profiler
 endif
 
-libgnunetcadettest_la_SOURCES = \
-  cadet_test_lib.c cadet_test_lib.h
-libgnunetcadettest_la_LIBADD = \
- $(top_builddir)/src/util/libgnunetutil.la \
- $(top_builddir)/src/testbed/libgnunettestbed.la \
- libgnunetcadet.la
-
 if HAVE_TESTING
 check_PROGRAMS = \
   test_cadet_local_mq \
-  test_cadet_2_forward_new \
-  test_cadet_2_forward_new \
-  test_cadet_2_signal_new \
-  test_cadet_2_keepalive_new \
-  test_cadet_2_speed_new \
-  test_cadet_2_speed_ack_new \
-  test_cadet_2_speed_backwards_new \
-  test_cadet_2_speed_reliable_new \
-  test_cadet_2_speed_reliable_backwards_new \
-  test_cadet_5_forward_new \
-  test_cadet_5_signal_new \
-  test_cadet_5_keepalive_new \
-  test_cadet_5_speed_new \
-  test_cadet_5_speed_ack_new \
-  test_cadet_5_speed_reliable_new \
-  test_cadet_5_speed_reliable_backwards_new \
-  test_cadet_5_speed_backwards_new \
-  test_cadet_single \
-  test_cadet_local \
+  test_cadet_2_forward \
   test_cadet_2_forward \
   test_cadet_2_signal \
   test_cadet_2_keepalive \
@@ -161,41 +100,39 @@ check_PROGRAMS = \
   test_cadet_5_speed_backwards
 endif
 
+
+#gnunet_cadet_profiler_SOURCES = \
+#  gnunet-cadet-profiler.c
+#gnunet_cadet_profiler_LDADD = $(ld_cadet_test_lib)
+
+
+test_cadet_local_mq_SOURCES = \
+  test_cadet_local_mq.c
+test_cadet_local_mq_LDADD = \
+  libgnunetcadet.la \
+  $(top_builddir)/src/testing/libgnunettesting.la \
+  $(top_builddir)/src/util/libgnunetutil.la
+
+
+libgnunetcadettest_la_SOURCES = \
+  cadet_test_lib.c cadet_test_lib.h
+libgnunetcadettest_la_LIBADD = \
+ $(top_builddir)/src/util/libgnunetutil.la \
+ $(top_builddir)/src/testbed/libgnunettestbed.la \
+ libgnunetcadet.la
+
 ld_cadet_test_lib = \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/testing/libgnunettesting.la \
-  libgnunetcadettest.la \
   libgnunetcadet.la \
+  libgnunetcadettest.la \
   $(top_builddir)/src/testbed/libgnunettestbed.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la
-
 dep_cadet_test_lib = \
   libgnunetcadet.la \
   libgnunetcadettest.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la
 
-
-gnunet_cadet_profiler_SOURCES = \
-  gnunet-cadet-profiler.c
-gnunet_cadet_profiler_LDADD = $(ld_cadet_test_lib)
-
-
-test_cadet_single_SOURCES = \
-  test_cadet_single.c
-test_cadet_single_LDADD = $(ld_cadet_test_lib)
-
-test_cadet_local_SOURCES = \
-  test_cadet_local.c
-test_cadet_local_LDADD = $(ld_cadet_test_lib)
-
-
-test_cadet_local_mq_SOURCES = \
-  test_cadet_local_mq.c
-test_cadet_local_mq_LDADD = \
-  libgnunetcadetnew.la \
-  $(top_builddir)/src/testing/libgnunettesting.la \
-  $(top_builddir)/src/util/libgnunetutil.la
-
 test_cadet_2_forward_SOURCES = \
   test_cadet.c
 test_cadet_2_forward_LDADD = $(ld_cadet_test_lib)
@@ -228,7 +165,6 @@ test_cadet_2_speed_reliable_backwards_SOURCES = \
   test_cadet.c
 test_cadet_2_speed_reliable_backwards_LDADD = $(ld_cadet_test_lib)
 
-
 test_cadet_5_forward_SOURCES = \
   test_cadet.c
 test_cadet_5_forward_LDADD = $(ld_cadet_test_lib)
@@ -262,92 +198,6 @@ test_cadet_5_speed_reliable_backwards_SOURCES = \
 test_cadet_5_speed_reliable_backwards_LDADD = $(ld_cadet_test_lib)
 
 
-# NEW TESTS
-libgnunetcadettestnew_la_SOURCES = \
-  cadet_test_lib_new.c cadet_test_lib_new.h
-libgnunetcadettestnew_la_LIBADD = \
- $(top_builddir)/src/util/libgnunetutil.la \
- $(top_builddir)/src/testbed/libgnunettestbed.la \
- libgnunetcadetnew.la
-
-ld_cadet_test_lib_new = \
-  $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/testing/libgnunettesting.la \
-  libgnunetcadetnew.la \
-  libgnunetcadettestnew.la \
-  $(top_builddir)/src/testbed/libgnunettestbed.la \
-  $(top_builddir)/src/statistics/libgnunetstatistics.la
-dep_cadet_test_lib_new = \
-  libgnunetcadetnew.la \
-  libgnunetcadettestnew.la \
-  $(top_builddir)/src/statistics/libgnunetstatistics.la
-
-test_cadet_2_forward_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_2_forward_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_2_signal_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_2_signal_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_2_keepalive_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_2_keepalive_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_2_speed_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_2_speed_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_2_speed_ack_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_2_speed_ack_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_2_speed_backwards_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_2_speed_backwards_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_2_speed_reliable_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_2_speed_reliable_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_2_speed_reliable_backwards_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_2_speed_reliable_backwards_new_LDADD = $(ld_cadet_test_lib_new)
-
-
-test_cadet_5_forward_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_5_forward_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_5_signal_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_5_signal_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_5_keepalive_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_5_keepalive_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_5_speed_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_5_speed_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_5_speed_ack_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_5_speed_ack_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_5_speed_backwards_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_5_speed_backwards_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_5_speed_reliable_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_5_speed_reliable_new_LDADD = $(ld_cadet_test_lib_new)
-
-test_cadet_5_speed_reliable_backwards_new_SOURCES = \
-  test_cadet_new.c
-test_cadet_5_speed_reliable_backwards_new_LDADD = $(ld_cadet_test_lib_new)
-
-
 if ENABLE_TEST_RUN
 AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
 TESTS = \
diff --git a/src/cadet/TODO b/src/cadet/TODO
index 820efab7a..06567b0ad 100644
--- a/src/cadet/TODO
+++ b/src/cadet/TODO
@@ -1,6 +1,10 @@
-- URGENT: Congestion/flow control (CHANNEL):
-  + estimate max bandwidth using bursts and use to for CONGESTION CONTROL!
-   (and figure out how/where to use this!)
+- URGENT:
+  + if 'client-not-ready', we do not ACK at all, and sender keeps
+    retransmitting again and again; would be good to do flow-control notification instead
+    of not ACKing that we got the data but are simply not ready for more!
+  + Congestion/flow control (CHANNEL):
+    estimate max bandwidth using bursts and use to for CONGESTION CONTROL!
+    (and figure out how/where to use this!)
 
 - HIGH: revisit handling of 'unbuffered' traffic! (CHANNEL/TUNNEL)
         (need to push down through tunnel into connection selection);
diff --git a/src/cadet/cadet.conf.in b/src/cadet/cadet.conf.in
index 86ba2e535..d50e168f0 100644
--- a/src/cadet/cadet.conf.in
+++ b/src/cadet/cadet.conf.in
@@ -3,7 +3,7 @@ FORCESTART = YES
 AUTOSTART = @AUTOSTART@
 @JAVAPORT@PORT = 2096
 HOSTNAME = localhost
-BINARY = gnunet-service-cadet-new
+BINARY = gnunet-service-cadet
 # PREFIX = valgrind --leak-check=yes
 ACCEPT_FROM = 127.0.0.1;
 ACCEPT_FROM6 = ::1;
diff --git a/src/cadet/cadet.h b/src/cadet/cadet.h
index 451d1f354..99f9f2653 100644
--- a/src/cadet/cadet.h
+++ b/src/cadet/cadet.h
@@ -59,7 +59,7 @@ extern "C"
 #include "gnunet_core_service.h"
 #include "gnunet_cadet_service.h"
 #include "gnunet_protocols.h"
-#include <gnunet_cadet_service.h>
+#include "gnunet_cadet_service.h"
 
 /******************************************************************************/
 /**************************       CONSTANTS      ******************************/
diff --git a/src/cadet/cadet_api.c b/src/cadet/cadet_api.c
index 7b9ac62b3..decf473a9 100644
--- a/src/cadet/cadet_api.c
+++ b/src/cadet/cadet_api.c
@@ -21,8 +21,8 @@
  * @file cadet/cadet_api.c
  * @brief cadet api: client implementation of cadet service
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
  */
-
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_constants.h"
@@ -32,57 +32,9 @@
 
 #define LOG(kind,...) GNUNET_log_from (kind, "cadet-api",__VA_ARGS__)
 
-/******************************************************************************/
-/************************      DATA STRUCTURES     ****************************/
-/******************************************************************************/
-
 /**
- * Transmission queue to the service
- *
- * @deprecated
+ * Ugly legacy hack.
  */
-struct GNUNET_CADET_TransmitHandle
-{
-  /**
-   * Double Linked list
-   */
-  struct GNUNET_CADET_TransmitHandle *next;
-
-  /**
-   * Double Linked list
-   */
-  struct GNUNET_CADET_TransmitHandle *prev;
-
-  /**
-   * Channel this message is sent on / for (may be NULL for control messages).
-   */
-  struct GNUNET_CADET_Channel *channel;
-
-  /**
-   * Request data task.
-   */
-  struct GNUNET_SCHEDULER_Task *request_data_task;
-
-  /**
-   * Callback to obtain the message to transmit, or NULL if we
-   * got the message in 'data'.  Notice that messages built
-   * by 'notify' need to be encapsulated with information about
-   * the 'target'.
-   */
-  GNUNET_CONNECTION_TransmitReadyNotify notify;
-
-  /**
-   * Closure for 'notify'
-   */
-  void *notify_cls;
-
-  /**
-   * Size of the payload.
-   */
-  size_t size;
-};
-
-
 union CadetInfoCB
 {
 
@@ -119,69 +71,19 @@ union CadetInfoCB
 struct GNUNET_CADET_Handle
 {
   /**
-   * Flag to indicate old or MQ API.
-   */
-  int mq_api;
-
-  /**
-   * Message queue (if available).
+   * Message queue.
    */
   struct GNUNET_MQ_Handle *mq;
 
   /**
-   * Set of handlers used for processing incoming messages in the channels
-   *
-   * @deprecated
-   */
-  const struct GNUNET_CADET_MessageHandler *message_handlers;
-
-  /**
-   * Number of handlers in the handlers array.
-   *
-   * @deprecated
-   */
-  unsigned int n_handlers;
-
-  /**
    * Ports open.
    */
   struct GNUNET_CONTAINER_MultiHashMap *ports;
 
   /**
-   * Double linked list of the channels this client is connected to, head.
+   * Channels open.
    */
-  struct GNUNET_CADET_Channel *channels_head;
-
-  /**
-   * Double linked list of the channels this client is connected to, tail.
-   */
-  struct GNUNET_CADET_Channel *channels_tail;
-
-  /**
-   * Callback for inbound channel disconnection
-   */
-  GNUNET_CADET_ChannelEndHandler *cleaner;
-
-  /**
-   * Closure for all the handlers given by the client
-   *
-   * @deprecated
-   */
-  void *cls;
-
-  /**
-   * Messages to send to the service, head.
-   *
-   * @deprecated
-   */
-  struct GNUNET_CADET_TransmitHandle *th_head;
-
-  /**
-   * Messages to send to the service, tail.
-   *
-   * @deprecated
-   */
-  struct GNUNET_CADET_TransmitHandle *th_tail;
+  struct GNUNET_CONTAINER_MultiHashMap32 *channels;
 
   /**
    * child of the next channel to create (to avoid reusing IDs often)
@@ -194,14 +96,9 @@ struct GNUNET_CADET_Handle
   const struct GNUNET_CONFIGURATION_Handle *cfg;
 
   /**
-   * Time to the next reconnect in case one reconnect fails
-   */
-  struct GNUNET_TIME_Relative reconnect_time;
-
-  /**
    * Task for trying to reconnect.
    */
-  struct GNUNET_SCHEDULER_Task * reconnect_task;
+  struct GNUNET_SCHEDULER_Task *reconnect_task;
 
   /**
    * Callback for an info task (only one active at a time).
@@ -212,23 +109,12 @@ struct GNUNET_CADET_Handle
    * Info callback closure for @c info_cb.
    */
   void *info_cls;
-};
-
 
-/**
- * Description of a peer
- */
-struct GNUNET_CADET_Peer
-{
   /**
-   * ID of the peer in short form
+   * Time to the next reconnect in case one reconnect fails
    */
-  GNUNET_PEER_Id id;
+  struct GNUNET_TIME_Relative reconnect_time;
 
-  /**
-   * Channel this peer belongs to
-   */
-  struct GNUNET_CADET_Channel *t;
 };
 
 
@@ -237,15 +123,11 @@ struct GNUNET_CADET_Peer
  */
 struct GNUNET_CADET_Channel
 {
-  /**
-   * DLL next
-   */
-  struct GNUNET_CADET_Channel *next;
 
   /**
-   * DLL prev
+   * Other end of the channel.
    */
-  struct GNUNET_CADET_Channel *prev;
+  struct GNUNET_PeerIdentity peer;
 
   /**
    * Handle to the cadet this channel belongs to
@@ -253,40 +135,18 @@ struct GNUNET_CADET_Channel
   struct GNUNET_CADET_Handle *cadet;
 
   /**
-   * Local ID of the channel
-   */
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  /**
    * Channel's port, if incoming.
    */
   struct GNUNET_CADET_Port *incoming_port;
 
   /**
-   * Other end of the channel.
-   */
-  GNUNET_PEER_Id peer;
-
-  /**
-   * Any data the caller wants to put in here
+   * Any data the caller wants to put in here, used for the
+   * various callbacks (@e disconnects, @e window_changes, handlers).
    */
   void *ctx;
 
   /**
-   * Channel options: reliability, etc.
-   */
-  enum GNUNET_CADET_ChannelOption options;
-
-  /**
-   * Are we allowed to send to the service?
-   *
-   * @deprecated?
-   */
-  unsigned int allow_send;
-
-  /*****************************    MQ     ************************************/
-  /**
-   * Message Queue for the channel.
+   * Message Queue for the channel (which we are implementing).
    */
   struct GNUNET_MQ_Handle *mq;
 
@@ -296,7 +156,9 @@ struct GNUNET_CADET_Channel
   struct GNUNET_SCHEDULER_Task *mq_cont;
 
   /**
-   * Pending envelope in case we don't have an ACK from the service.
+   * Pending envelope with a message to be transmitted to the
+   * service as soon as we are allowed to.  Should only be
+   * non-NULL if @e allow_send is 0.
    */
   struct GNUNET_MQ_Envelope *pending_env;
 
@@ -310,6 +172,21 @@ struct GNUNET_CADET_Channel
    */
   GNUNET_CADET_DisconnectEventHandler disconnects;
 
+  /**
+   * Local ID of the channel, #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI bit is set if outbound.
+   */
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  /**
+   * Channel options: reliability, etc.
+   */
+  enum GNUNET_CADET_ChannelOption options;
+
+  /**
+   * How many messages are we allowed to send to the service right now?
+   */
+  unsigned int allow_send;
+
 };
 
 
@@ -318,35 +195,22 @@ struct GNUNET_CADET_Channel
  */
 struct GNUNET_CADET_Port
 {
-  /**
-   * Handle to the CADET session this port belongs to.
-   */
-  struct GNUNET_CADET_Handle *cadet;
 
   /**
-   * Port ID.
-   *
-   * @deprecated
+   * Port "number"
    */
-  struct GNUNET_HashCode *hash;
+  struct GNUNET_HashCode id;
 
   /**
-   * Callback handler for incoming channels on this port.
+   * Handle to the CADET session this port belongs to.
    */
-  GNUNET_CADET_InboundChannelNotificationHandler *handler;
+  struct GNUNET_CADET_Handle *cadet;
 
   /**
    * Closure for @a handler.
    */
   void *cls;
 
-  /*****************************    MQ     ************************************/
-
-  /**
-   * Port "number"
-   */
-  struct GNUNET_HashCode id;
-
   /**
    * Handler for incoming channels on this port
    */
@@ -355,7 +219,7 @@ struct GNUNET_CADET_Port
   /**
    * Closure for @ref connects
    */
-  void * connects_cls;
+  void *connects_cls;
 
   /**
    * Window size change handler.
@@ -363,106 +227,30 @@ struct GNUNET_CADET_Port
   GNUNET_CADET_WindowSizeEventHandler window_changes;
 
   /**
-   * Handler called when an incoming channel is destroyed..
+   * Handler called when an incoming channel is destroyed.
    */
   GNUNET_CADET_DisconnectEventHandler disconnects;
 
   /**
    * Payload handlers for incoming channels.
    */
-  const struct GNUNET_MQ_MessageHandler *handlers;
+  struct GNUNET_MQ_MessageHandler *handlers;
 };
 
 
 /**
- * Implementation state for cadet's message queue.
- */
-struct CadetMQState
-{
-  /**
-   * The current transmit handle, or NULL
-   * if no transmit is active.
-   */
-  struct GNUNET_CADET_TransmitHandle *th;
-
-  /**
-   * Channel to send the data over.
-   */
-  struct GNUNET_CADET_Channel *channel;
-};
-
-
-
-/******************************************************************************/
-/*********************      FUNCTION DECLARATIONS     *************************/
-/******************************************************************************/
-
-/**
- * Reconnect to the service, retransmit all infomation to try to restore the
- * original state.
- *
- * @param h Handle to the CADET service.
- */
-static void
-schedule_reconnect (struct GNUNET_CADET_Handle *h);
-
-
-/**
- * Reconnect callback: tries to reconnect again after a failer previous
- * reconnection.
- *
- * @param cls Closure (cadet handle).
- */
-static void
-reconnect_cbk (void *cls);
-
-
-/**
- * Reconnect to the service, retransmit all infomation to try to restore the
- * original state.
- *
- * @param h handle to the cadet
- */
-static void
-reconnect (struct GNUNET_CADET_Handle *h);
-
-
-/******************************************************************************/
-/***********************     AUXILIARY FUNCTIONS      *************************/
-/******************************************************************************/
-
-/**
- * Check if transmission is a payload packet.
- *
- * @param th Transmission handle.
- *
- * @return #GNUNET_YES if it is a payload packet,
- *         #GNUNET_NO if it is a cadet management packet.
- */
-static int
-th_is_payload (struct GNUNET_CADET_TransmitHandle *th)
-{
-  return (th->notify != NULL) ? GNUNET_YES : GNUNET_NO;
-}
-
-
-/**
  * Find the Port struct for a hash.
  *
  * @param h CADET handle.
  * @param hash HashCode for the port number.
- *
  * @return The port handle if known, NULL otherwise.
  */
 static struct GNUNET_CADET_Port *
 find_port (const struct GNUNET_CADET_Handle *h,
            const struct GNUNET_HashCode *hash)
 {
-  struct GNUNET_CADET_Port *p;
-
-  p = GNUNET_CONTAINER_multihashmap_get (h->ports, hash);
-
-  return p;
+  return GNUNET_CONTAINER_multihashmap_get (h->ports,
+                                            hash);
 }
 
 
@@ -474,15 +262,11 @@ find_port (const struct GNUNET_CADET_Handle *h,
  * @return handle to the required channel or NULL if not found
  */
 static struct GNUNET_CADET_Channel *
-retrieve_channel (struct GNUNET_CADET_Handle *h,
-                  struct GNUNET_CADET_ClientChannelNumber ccn)
+find_channel (struct GNUNET_CADET_Handle *h,
+              struct GNUNET_CADET_ClientChannelNumber ccn)
 {
-  struct GNUNET_CADET_Channel *ch;
-
-  for (ch = h->channels_head; NULL != ch; ch = ch->next)
-    if (ch->ccn.channel_of_client == ccn.channel_of_client)
-      return ch;
-  return NULL;
+  return GNUNET_CONTAINER_multihashmap32_get (h->channels,
+                                              ntohl (ccn.channel_of_client));
 }
 
 
@@ -490,38 +274,37 @@ retrieve_channel (struct GNUNET_CADET_Handle *h,
  * Create a new channel and insert it in the channel list of the cadet handle
  *
  * @param h Cadet handle
- * @param ccn Desired ccn of the channel, 0 to assign one automatically.
- *
+ * @param ccnp pointer to desired ccn of the channel, NULL to assign one automatically.
  * @return Handle to the created channel.
  */
 static struct GNUNET_CADET_Channel *
 create_channel (struct GNUNET_CADET_Handle *h,
-                struct GNUNET_CADET_ClientChannelNumber ccn)
+                const struct GNUNET_CADET_ClientChannelNumber *ccnp)
 {
   struct GNUNET_CADET_Channel *ch;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
 
   ch = GNUNET_new (struct GNUNET_CADET_Channel);
-  GNUNET_CONTAINER_DLL_insert (h->channels_head,
-                               h->channels_tail,
-                               ch);
   ch->cadet = h;
-  if (0 == ccn.channel_of_client)
+  if (NULL == ccnp)
   {
-    ch->ccn = h->next_ccn;
-    while (NULL != retrieve_channel (h,
-                                     h->next_ccn))
-    {
+    while (NULL !=
+           find_channel (h,
+                         h->next_ccn))
       h->next_ccn.channel_of_client
-        = htonl (1 + ntohl (h->next_ccn.channel_of_client));
-      if (0 == ntohl (h->next_ccn.channel_of_client))
-        h->next_ccn.channel_of_client
-          = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-    }
+        = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI | (1 + ntohl (h->next_ccn.channel_of_client)));
+    ccn = h->next_ccn;
   }
   else
   {
-    ch->ccn = ccn;
+    ccn = *ccnp;
   }
+  ch->ccn = ccn;
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap32_put (h->channels,
+                                                      ntohl (ch->ccn.channel_of_client),
+                                                      ch,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   return ch;
 }
 
@@ -535,114 +318,106 @@ create_channel (struct GNUNET_CADET_Handle *h,
  *
  * @param ch Pointer to the channel.
  * @param call_cleaner Whether to call the cleaner handler.
- *
- * @return Handle to the required channel or NULL if not found.
  */
 static void
 destroy_channel (struct GNUNET_CADET_Channel *ch)
 {
-  struct GNUNET_CADET_Handle *h;
-  struct GNUNET_CADET_TransmitHandle *th;
-  struct GNUNET_CADET_TransmitHandle *next;
+  struct GNUNET_CADET_Handle *h = ch->cadet;
 
-  if (NULL == ch)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  h = ch->cadet;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " destroy_channel %X of %p\n",
+       "Destroying channel %X of %p\n",
        ch->ccn,
        h);
-
-  GNUNET_CONTAINER_DLL_remove (h->channels_head,
-                               h->channels_tail,
-                               ch);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (h->channels,
+                                                         ntohl (ch->ccn.channel_of_client),
+                                                         ch));
   if (NULL != ch->mq_cont)
   {
     GNUNET_SCHEDULER_cancel (ch->mq_cont);
     ch->mq_cont = NULL;
   }
   /* signal channel destruction */
-  if (0 != ch->peer)
-  {
-    if (NULL != h->cleaner)
-    {
-      /** @a deprecated  */
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-          " calling cleaner\n");
-      h->cleaner (h->cls, ch, ch->ctx);
-    }
-    else if (NULL != ch->disconnects)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-          " calling disconnect handler\n");
-      ch->disconnects (ch->ctx, ch);
-    }
-    else
-    {
-      /* Application won't be aware of the channel destruction and use
-       * a pointer to free'd memory.
-       */
-      GNUNET_assert (0);
-    }
-  }
-
-  /* check that clients did not leave messages behind in the queue */
-  for (th = h->th_head; NULL != th; th = next)
-  {
-    next = th->next;
-    if (th->channel != ch)
-      continue;
-    /* Clients should have aborted their requests already.
-     * Management traffic should be ok, as clients can't cancel that.
-     * If the service crashed and we are reconnecting, it's ok.
-     */
-    GNUNET_break (GNUNET_NO == th_is_payload (th));
-    GNUNET_CADET_notify_transmit_ready_cancel (th);
-  }
-
-  if (0 != ch->peer)
-    GNUNET_PEER_change_rc (ch->peer, -1);
+  if (NULL != ch->disconnects)
+    ch->disconnects (ch->ctx,
+                     ch);
+  if (NULL != ch->pending_env)
+    GNUNET_MQ_discard (ch->pending_env);
+  GNUNET_MQ_destroy (ch->mq);
   GNUNET_free (ch);
 }
 
 
 /**
- * Add a transmit handle to the transmission queue and set the
- * timeout if needed.
+ * Reconnect to the service, retransmit all infomation to try to restore the
+ * original state.
+ *
+ * @param h handle to the cadet
+ */
+static void
+reconnect (struct GNUNET_CADET_Handle *h);
+
+
+/**
+ * Reconnect callback: tries to reconnect again after a failer previous
+ * reconnecttion
  *
- * @param h cadet handle with the queue head and tail
- * @param th handle to the packet to be transmitted
+ * @param cls closure (cadet handle)
  */
 static void
-add_to_queue (struct GNUNET_CADET_Handle *h,
-              struct GNUNET_CADET_TransmitHandle *th)
+reconnect_cbk (void *cls)
+{
+  struct GNUNET_CADET_Handle *h = cls;
+
+  h->reconnect_task = NULL;
+  reconnect (h);
+}
+
+
+/**
+ * Function called during #reconnect() to destroy
+ * all channels that are still open.
+ *
+ * @param cls the `struct GNUNET_CADET_Handle`
+ * @param cid chanenl ID
+ * @param value a `struct GNUNET_CADET_Channel` to destroy
+ * @return #GNUNET_OK (continue to iterate)
+ */
+static int
+destroy_channel_on_reconnect_cb (void *cls,
+                                 uint32_t cid,
+                                 void *value)
 {
-  GNUNET_CONTAINER_DLL_insert_tail (h->th_head,
-                                    h->th_tail,
-                                    th);
+  /* struct GNUNET_CADET_Handle *handle = cls; */
+  struct GNUNET_CADET_Channel *ch = value;
+
+  destroy_channel (ch);
+  return GNUNET_OK;
 }
 
 
 /**
- * Remove a transmit handle from the transmission queue, if present.
+ * Reconnect to the service, retransmit all infomation to try to restore the
+ * original state.
  *
- * Safe to call even if not queued.
+ * @param h handle to the cadet
  *
- * @param th handle to the packet to be unqueued.
+ * @return #GNUNET_YES in case of sucess, #GNUNET_NO otherwise (service down...)
  */
 static void
-remove_from_queue (struct GNUNET_CADET_TransmitHandle *th)
+schedule_reconnect (struct GNUNET_CADET_Handle *h)
 {
-  struct GNUNET_CADET_Handle *h = th->channel->cadet;
-
-  /* It might or might not have been queued (rarely not), but check anyway. */
-  if (NULL != th->next || h->th_tail == th)
-  {
-    GNUNET_CONTAINER_DLL_remove (h->th_head, h->th_tail, th);
-  }
+  if (NULL != h->reconnect_task)
+    return;
+  GNUNET_CONTAINER_multihashmap32_iterate (h->channels,
+                                           &destroy_channel_on_reconnect_cb,
+                                           h);
+  h->reconnect_task
+    = GNUNET_SCHEDULER_add_delayed (h->reconnect_time,
+                                    &reconnect_cbk,
+                                    h);
+  h->reconnect_time
+    = GNUNET_TIME_STD_BACKOFF (h->reconnect_time);
 }
 
 
@@ -655,29 +430,44 @@ static void
 notify_window_size (struct GNUNET_CADET_Channel *ch)
 {
   if (NULL != ch->window_changes)
-  {
-    ch->window_changes (ch->ctx, ch, ch->allow_send);
-  }
+    ch->window_changes (ch->ctx,
+                        ch, /* FIXME: remove 'ch'? */
+                        ch->allow_send);
 }
 
-/******************************************************************************/
-/***********************      MQ API CALLBACKS     ****************************/
-/******************************************************************************/
 
 /**
- * Allow the MQ implementation to send the next message.
+ * Transmit the next message from our queue.
  *
  * @param cls Closure (channel whose mq to activate).
  */
 static void
-cadet_mq_send_continue (void *cls)
+cadet_mq_send_now (void *cls)
 {
   struct GNUNET_CADET_Channel *ch = cls;
+  struct GNUNET_MQ_Envelope *env = ch->pending_env;
 
   ch->mq_cont = NULL;
+  if (0 == ch->allow_send)
+  {
+    /* how did we get here? */
+    GNUNET_break (0);
+    return;
+  }
+  if (NULL == env)
+  {
+    /* how did we get here? */
+    GNUNET_break (0);
+    return;
+  }
+  ch->allow_send--;
+  ch->pending_env = NULL;
+  GNUNET_MQ_send (ch->cadet->mq,
+                  env);
   GNUNET_MQ_impl_send_continue (ch->mq);
 }
 
+
 /**
  * Implement sending functionality of a message queue for
  * us sending messages to a peer.
@@ -701,7 +491,6 @@ cadet_mq_send_impl (struct GNUNET_MQ_Handle *mq,
   struct GNUNET_MQ_Envelope *env;
   struct GNUNET_CADET_LocalData *cadet_msg;
 
-
   if (NULL == h->mq)
   {
     /* We're currently reconnecting, pretend this worked */
@@ -717,26 +506,16 @@ cadet_mq_send_impl (struct GNUNET_MQ_Handle *mq,
     GNUNET_MQ_impl_send_continue (mq);
     return;
   }
-
   env = GNUNET_MQ_msg_nested_mh (cadet_msg,
                                  GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
                                  msg);
   cadet_msg->ccn = ch->ccn;
-
+  GNUNET_assert (NULL == ch->pending_env);
+  ch->pending_env = env;
   if (0 < ch->allow_send)
-  {
-    /* Service has allowed this message, just send it and continue accepting */
-    GNUNET_MQ_send (h->mq, env);
-    ch->allow_send--;
-    ch->mq_cont = GNUNET_SCHEDULER_add_now (&cadet_mq_send_continue, ch);
-    // notify_window_size (ch); /* FIXME add "verbose" setting? */
-  }
-  else
-  {
-    /* Service has NOT allowed this message, queue it and wait for an ACK */
-    GNUNET_assert (NULL == ch->pending_env);
-    ch->pending_env = env;
-  }
+    ch->mq_cont
+      = GNUNET_SCHEDULER_add_now (&cadet_mq_send_now,
+                                  ch);
 }
 
 
@@ -763,14 +542,25 @@ cadet_mq_destroy_impl (struct GNUNET_MQ_Handle *mq,
  * the CADET service.  We should just complain about it but otherwise
  * continue processing.
  *
- * @param cls closure
+ * @param cls closure with our `struct GNUNET_CADET_Channel`
  * @param error error code
  */
 static void
 cadet_mq_error_handler (void *cls,
                         enum GNUNET_MQ_Error error)
 {
-  GNUNET_break_op (0);
+  struct GNUNET_CADET_Channel *ch = cls;
+
+  GNUNET_break (0);
+  if (GNUNET_MQ_ERROR_NO_MATCH == error)
+  {
+    /* Got a message we did not understand, still try to continue! */
+    GNUNET_CADET_receive_done (ch);
+  }
+  else
+  {
+    schedule_reconnect (ch->cadet);
+  }
 }
 
 
@@ -781,65 +571,20 @@ cadet_mq_error_handler (void *cls,
  * @param mq message queue
  * @param impl_state state specific to the implementation
  */
-
 static void
 cadet_mq_cancel_impl (struct GNUNET_MQ_Handle *mq,
                      void *impl_state)
 {
   struct GNUNET_CADET_Channel *ch = impl_state;
 
-  LOG (GNUNET_ERROR_TYPE_WARNING,
-       "Cannot cancel mq message on channel %X of %p\n",
-       ch->ccn.channel_of_client, ch->cadet);
-
-  GNUNET_break (0);
-}
-
-
-/******************************************************************************/
-/***********************      RECEIVE HANDLERS     ****************************/
-/******************************************************************************/
-
-
-/**
- * Call the @a notify callback given to #GNUNET_CADET_notify_transmit_ready to
- * request the data to send over MQ. Since MQ manages the queue, this function
- * is scheduled immediatly after a transmit ready notification.
- *
- * @param cls Closure (transmit handle).
- */
-static void
-request_data (void *cls)
-{
-  struct GNUNET_CADET_TransmitHandle *th = cls;
-  struct GNUNET_CADET_LocalData *msg;
-  struct GNUNET_MQ_Envelope *env;
-  size_t osize;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Requesting Data: %u bytes (allow send is %u)\n",
-       th->size,
-       th->channel->allow_send);
-
-  GNUNET_assert (0 < th->channel->allow_send);
-  th->channel->allow_send--;
-  /* NOTE: we may be allowed to send another packet immediately,
-     albeit the current logic waits for the ACK. */
-  th->request_data_task = NULL;
-  remove_from_queue (th);
-
-  env = GNUNET_MQ_msg_extra (msg,
-                             th->size,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
-  msg->ccn = th->channel->ccn;
-  osize = th->notify (th->notify_cls,
-                      th->size,
-                      &msg[1]);
-  GNUNET_assert (osize == th->size);
-
-  GNUNET_MQ_send (th->channel->cadet->mq,
-                  env);
-  GNUNET_free (th);
+  GNUNET_assert (NULL != ch->pending_env);
+  GNUNET_MQ_discard (ch->pending_env);
+  ch->pending_env = NULL;
+  if (NULL != ch->mq_cont)
+  {
+    GNUNET_SCHEDULER_cancel (ch->mq_cont);
+    ch->mq_cont = NULL;
+  }
 }
 
 
@@ -866,7 +611,8 @@ handle_channel_created (void *cls,
     GNUNET_break (0);
     return;
   }
-  port = find_port (h, port_number);
+  port = find_port (h,
+                    port_number);
   if (NULL == port)
   {
     /* We could have closed the port but the service didn't know about it yet
@@ -875,7 +621,6 @@ handle_channel_created (void *cls,
     struct GNUNET_CADET_LocalChannelDestroyMessage *d_msg;
     struct GNUNET_MQ_Envelope *env;
 
-    GNUNET_break (0);
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "No handler for incoming channel %X (on port %s, recently closed?)\n",
          ntohl (ccn.channel_of_client),
@@ -889,10 +634,9 @@ handle_channel_created (void *cls,
   }
 
   ch = create_channel (h,
-                       ccn);
-  ch->peer = GNUNET_PEER_intern (&msg->peer);
+                       &ccn);
+  ch->peer = msg->peer;
   ch->cadet = h;
-  ch->ccn = ccn;
   ch->incoming_port = port;
   ch->options = ntohl (msg->opt);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
@@ -901,34 +645,21 @@ handle_channel_created (void *cls,
        GNUNET_h2s (port_number),
        ch);
 
-  if (NULL != port->handler)
-  {
-    /** @deprecated */
-    /* Old style API */
-    ch->ctx = port->handler (port->cls,
-                             ch,
-                             &msg->peer,
-                             port->hash,
-                             ch->options);
-  }
-  else
-  {
-    /* MQ API */
-    GNUNET_assert (NULL != port->connects);
-    ch->window_changes = port->window_changes;
-    ch->disconnects = port->disconnects;
-    ch->mq = GNUNET_MQ_queue_for_callbacks (&cadet_mq_send_impl,
-                                            &cadet_mq_destroy_impl,
-                                            &cadet_mq_cancel_impl,
-                                            ch,
-                                            port->handlers,
-                                            &cadet_mq_error_handler,
-                                            ch);
-    ch->ctx = port->connects (port->cls,
-                              ch,
-                              &msg->peer);
-    GNUNET_MQ_set_handlers_closure (ch->mq, ch->ctx);
-  }
+  GNUNET_assert (NULL != port->connects);
+  ch->window_changes = port->window_changes;
+  ch->disconnects = port->disconnects;
+  ch->mq = GNUNET_MQ_queue_for_callbacks (&cadet_mq_send_impl,
+                                          &cadet_mq_destroy_impl,
+                                          &cadet_mq_cancel_impl,
+                                          ch,
+                                          port->handlers,
+                                          &cadet_mq_error_handler,
+                                          ch);
+  ch->ctx = port->connects (port->cls,
+                            ch,
+                            &msg->peer);
+  GNUNET_MQ_set_handlers_closure (ch->mq,
+                                  ch->ctx);
 }
 
 
@@ -944,22 +675,19 @@ handle_channel_destroy (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
   struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  ccn = msg->ccn;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Channel %X Destroy from service\n",
-       ntohl (ccn.channel_of_client));
-  ch = retrieve_channel (h,
-                         ccn);
 
+  ch = find_channel (h,
+                     msg->ccn);
   if (NULL == ch)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "channel %X unknown\n",
-         ntohl (ccn.channel_of_client));
+         "Received channel destroy for unknown channel %X from CADET service (recently close?)\n",
+         ntohl (msg->ccn.channel_of_client));
     return;
   }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received channel destroy for channel %X from CADET service\n",
+       ntohl (msg->ccn.channel_of_client));
   destroy_channel (ch);
 }
 
@@ -976,25 +704,14 @@ static int
 check_local_data (void *cls,
                   const struct GNUNET_CADET_LocalData *message)
 {
-  struct GNUNET_CADET_Handle *h = cls;
-  struct GNUNET_CADET_Channel *ch;
   uint16_t size;
 
   size = ntohs (message->header.size);
   if (sizeof (*message) + sizeof (struct GNUNET_MessageHeader) > size)
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-
-  ch = retrieve_channel (h,
-                         message->ccn);
-  if (NULL == ch)
-  {
-    GNUNET_break_op (0);
+    GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-
   return GNUNET_OK;
 }
 
@@ -1011,62 +728,31 @@ handle_local_data (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
   const struct GNUNET_MessageHeader *payload;
-  const struct GNUNET_CADET_MessageHandler *handler;
   struct GNUNET_CADET_Channel *ch;
   uint16_t type;
   int fwd;
 
-  ch = retrieve_channel (h,
-                         message->ccn);
+  ch = find_channel (h,
+                     message->ccn);
   if (NULL == ch)
   {
-    GNUNET_break_op (0);
-    reconnect (h);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Unknown channel %X for incoming data (recently closed?)\n",
+         ntohl (message->ccn.channel_of_client));
     return;
   }
 
-  payload = (struct GNUNET_MessageHeader *) &message[1];
+  payload = (const struct GNUNET_MessageHeader *) &message[1];
   type = ntohs (payload->type);
   fwd = ntohl (ch->ccn.channel_of_client) <= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got a %s data on channel %s [%X] of type %s (%u)\n",
-       GC_f2s (fwd),
-       GNUNET_i2s (GNUNET_PEER_resolve2 (ch->peer)),
+       "Got a %s data on channel %s [%X] of type %u\n",
+       fwd ? "FWD" : "BWD",
+       GNUNET_i2s (&ch->peer),
        ntohl (message->ccn.channel_of_client),
-       GC_m2s (type),
        type);
-  if (NULL != ch->mq)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "injecting msg %s into mq %p\n",
-         GC_m2s (ntohs (payload->type)),
-         ch->mq);
-    GNUNET_MQ_inject_message (ch->mq, payload);
-    return;
-  }
-  /** @a deprecated */
-  for (unsigned i=0;i<h->n_handlers;i++)
-  {
-    handler = &h->message_handlers[i];
-    if (handler->type == type)
-    {
-      if (GNUNET_OK !=
-          handler->callback (h->cls,
-                             ch,
-                             &ch->ctx,
-                             payload))
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "callback caused disconnection\n");
-        GNUNET_CADET_channel_destroy (ch);
-        return;
-      }
-      return;
-    }
-  }
-  /* Other peer sent message we do not comprehend. */
-  GNUNET_break_op (0);
-  GNUNET_CADET_receive_done (ch);
+  GNUNET_MQ_inject_message (ch->mq,
+                            payload);
 }
 
 
@@ -1083,55 +769,34 @@ handle_local_ack (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
   struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-  struct GNUNET_CADET_TransmitHandle *th;
 
-  ccn = message->ccn;
-  ch = retrieve_channel (h, ccn);
+  ch = find_channel (h,
+                     message->ccn);
   if (NULL == ch)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "ACK on unknown channel %X\n",
-         ntohl (ccn.channel_of_client));
+         ntohl (message->ccn.channel_of_client));
     return;
   }
   ch->allow_send++;
-  if (NULL != ch->mq)
+  if (NULL == ch->pending_env)
   {
-    if (NULL == ch->pending_env)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Got an ACK on mq channel %X, allow send now %u!\n",
-           ntohl (ch->ccn.channel_of_client),
-           ch->allow_send);
-      notify_window_size (ch);
-    }
-    else
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Got an ACK on mq channel %X, sending pending message!\n",
-           ntohl (ch->ccn.channel_of_client));
-      GNUNET_MQ_send (h->mq, ch->pending_env);
-      ch->allow_send--;
-      ch->pending_env = NULL;
-      ch->mq_cont = GNUNET_SCHEDULER_add_now (&cadet_mq_send_continue, ch);
-    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got an ACK on mq channel %X, allow send now %u!\n",
+         ntohl (ch->ccn.channel_of_client),
+         ch->allow_send);
+    notify_window_size (ch);
     return;
   }
-
-  /** @deprecated */
-  /* Old style API */
-  for (th = h->th_head; NULL != th; th = th->next)
-  {
-    if ( (th->channel == ch) &&
-         (NULL == th->request_data_task) )
-    {
-      th->request_data_task
-        = GNUNET_SCHEDULER_add_now (&request_data,
-                                    th);
-      break;
-    }
-  }
+  if (NULL != ch->mq_cont)
+    return; /* already working on it! */
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got an ACK on mq channel %X, sending pending message!\n",
+       ntohl (ch->ccn.channel_of_client));
+  ch->mq_cont
+    = GNUNET_SCHEDULER_add_now (&cadet_mq_send_now,
+                                ch);
 }
 
 
@@ -1149,134 +814,15 @@ handle_mq_error (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MQ ERROR: %u\n", error);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "MQ ERROR: %u\n",
+              error);
   GNUNET_MQ_destroy (h->mq);
   h->mq = NULL;
   reconnect (h);
 }
 
 
-/*
- * Process a local reply about info on all channels, pass info to the user.
- *
- * @param h Cadet handle.
- * @param message Message itself.
- */
-// static void
-// process_get_channels (struct GNUNET_CADET_Handle *h,
-//                      const struct GNUNET_MessageHeader *message)
-// {
-//   struct GNUNET_CADET_LocalInfo *msg;
-//
-//   GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Get Channels messasge received\n");
-//
-//   if (NULL == h->channels_cb)
-//   {
-//     GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "  ignored\n");
-//     return;
-//   }
-//
-//   msg = (struct GNUNET_CADET_LocalInfo *) message;
-//   if (ntohs (message->size) !=
-//       (sizeof (struct GNUNET_CADET_LocalInfo) +
-//        sizeof (struct GNUNET_PeerIdentity)))
-//   {
-//     GNUNET_break_op (0);
-//     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-//                 "Get channels message: size %hu - expected %u\n",
-//                 ntohs (message->size),
-//                 sizeof (struct GNUNET_CADET_LocalInfo));
-//     return;
-//   }
-//   h->channels_cb (h->channels_cls,
-//                   ntohl (msg->channel_id),
-//                   &msg->owner,
-//                   &msg->destination);
-// }
-
-
-
-/*
- * Process a local monitor_channel reply, pass info to the user.
- *
- * @param h Cadet handle.
- * @param message Message itself.
- */
-// static void
-// process_show_channel (struct GNUNET_CADET_Handle *h,
-//                      const struct GNUNET_MessageHeader *message)
-// {
-//   struct GNUNET_CADET_LocalInfo *msg;
-//   size_t esize;
-//
-//   GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Show Channel messasge received\n");
-//
-//   if (NULL == h->channel_cb)
-//   {
-//     GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "  ignored\n");
-//     return;
-//   }
-//
-//   /* Verify message sanity */
-//   msg = (struct GNUNET_CADET_LocalInfo *) message;
-//   esize = sizeof (struct GNUNET_CADET_LocalInfo);
-//   if (ntohs (message->size) != esize)
-//   {
-//     GNUNET_break_op (0);
-//     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-//                 "Show channel message: size %hu - expected %u\n",
-//                 ntohs (message->size),
-//                 esize);
-//
-//     h->channel_cb (h->channel_cls, NULL, NULL);
-//     h->channel_cb = NULL;
-//     h->channel_cls = NULL;
-//
-//     return;
-//   }
-//
-//   h->channel_cb (h->channel_cls,
-//                  &msg->destination,
-//                  &msg->owner);
-// }
-
-
-
-/**
- * Check that message received from CADET service is well-formed.
- *
- * @param cls the `struct GNUNET_CADET_Handle`
- * @param message the message we got
- * @return #GNUNET_OK if the message is well-formed,
- *         #GNUNET_SYSERR otherwise
- */
-static int
-check_get_peers (void *cls,
-                 const struct GNUNET_CADET_LocalInfoPeer *message)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  uint16_t size;
-
-  if (NULL == h->info_cb.peers_cb)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "  no handler for peesr monitor message!\n");
-    return GNUNET_SYSERR;
-  }
-
-  size = ntohs (message->header.size);
-  if (sizeof (struct GNUNET_CADET_LocalInfoPeer) > size)
-  {
-    h->info_cb.peers_cb (h->info_cls, NULL, -1, 0, 0);
-    h->info_cb.peers_cb = NULL;
-    h->info_cls = NULL;
-    return GNUNET_SYSERR;
-  }
-
-  return GNUNET_OK;
-}
-
-
 /**
  * Process a local reply about info on all tunnels, pass info to the user.
  *
@@ -1288,9 +834,13 @@ handle_get_peers (void *cls,
                   const struct GNUNET_CADET_LocalInfoPeer *msg)
 {
   struct GNUNET_CADET_Handle *h = cls;
-  h->info_cb.peers_cb (h->info_cls, &msg->destination,
+
+  if (NULL == h->info_cb.peers_cb)
+    return;
+  h->info_cb.peers_cb (h->info_cls,
+                       &msg->destination,
                        (int) ntohs (msg->tunnel),
-                       (unsigned int ) ntohs (msg->paths),
+                       (unsigned int) ntohs (msg->paths),
                        0);
 }
 
@@ -1307,62 +857,39 @@ static int
 check_get_peer (void *cls,
                 const struct GNUNET_CADET_LocalInfoPeer *message)
 {
-  struct GNUNET_CADET_Handle *h = cls;
-  const size_t msize = sizeof (struct GNUNET_CADET_LocalInfoPeer);
-  struct GNUNET_PeerIdentity *paths_array;
+  size_t msize = sizeof (struct GNUNET_CADET_LocalInfoPeer);
+  const struct GNUNET_PeerIdentity *paths_array;
   size_t esize;
   unsigned int epaths;
   unsigned int paths;
   unsigned int peers;
 
-  if (NULL == h->info_cb.peer_cb)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "  no handler for peer monitor message!\n");
-    goto clean_cls;
-  }
-
-  /* Verify message sanity */
   esize = ntohs (message->header.size);
   if (esize < msize)
   {
-    GNUNET_break_op (0);
-    h->info_cb.peer_cb (h->info_cls, NULL, 0, 0, 0, NULL);
-    goto clean_cls;
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
   if (0 != ((esize - msize) % sizeof (struct GNUNET_PeerIdentity)))
   {
-    GNUNET_break_op (0);
-    h->info_cb.peer_cb (h->info_cls, NULL, 0, 0, 0, NULL);
-    goto clean_cls;
-
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
   peers = (esize - msize) / sizeof (struct GNUNET_PeerIdentity);
-  epaths = (unsigned int) ntohs (message->paths);
-  paths_array = (struct GNUNET_PeerIdentity *) &message[1];
+  epaths = ntohs (message->paths);
+  paths_array = (const struct GNUNET_PeerIdentity *) &message[1];
   paths = 0;
-  for (int i = 0; i < peers; i++)
-  {
-    if (0 == memcmp (&paths_array[i], &message->destination,
+  for (unsigned int i = 0; i < peers; i++)
+    if (0 == memcmp (&paths_array[i],
+                     &message->destination,
                      sizeof (struct GNUNET_PeerIdentity)))
-    {
       paths++;
-    }
-  }
   if (paths != epaths)
   {
-    GNUNET_break_op (0);
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "p:%u, e: %u\n", paths, epaths);
-    h->info_cb.peer_cb (h->info_cls, NULL, 0, 0, 0, NULL);
-    goto clean_cls;
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
-
   return GNUNET_OK;
-
-clean_cls:
-  h->info_cb.peer_cb = NULL;
-  h->info_cls = NULL;
-  return GNUNET_SYSERR;
 }
 
 
@@ -1377,22 +904,26 @@ handle_get_peer (void *cls,
                  const struct GNUNET_CADET_LocalInfoPeer *message)
 {
   struct GNUNET_CADET_Handle *h = cls;
-  struct GNUNET_PeerIdentity *paths_array;
+  const struct GNUNET_PeerIdentity *paths_array;
   unsigned int paths;
   unsigned int path_length;
   int neighbor;
   unsigned int peers;
 
-  paths = (unsigned int) ntohs (message->paths);
-  paths_array = (struct GNUNET_PeerIdentity *) &message[1];
+  if (NULL == h->info_cb.peer_cb)
+    return;
+  paths = ntohs (message->paths);
+  paths_array = (const struct GNUNET_PeerIdentity *) &message[1];
   peers = (ntohs (message->header.size) - sizeof (*message))
           / sizeof (struct GNUNET_PeerIdentity);
   path_length = 0;
   neighbor = GNUNET_NO;
 
-  for (int i = 0; i < peers; i++)
+  for (unsigned int i = 0; i < peers; i++)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " %s\n", GNUNET_i2s (&paths_array[i]));
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                " %s\n",
+                GNUNET_i2s (&paths_array[i]));
     path_length++;
     if (0 == memcmp (&paths_array[i], &message->destination,
                      sizeof (struct GNUNET_PeerIdentity)))
@@ -1404,7 +935,7 @@ handle_get_peer (void *cls,
   }
 
   /* Call Callback with tunnel info. */
-  paths_array = (struct GNUNET_PeerIdentity *) &message[1];
+  paths_array = (const struct GNUNET_PeerIdentity *) &message[1];
   h->info_cb.peer_cb (h->info_cls,
                       &message->destination,
                       (int) ntohs (message->tunnel),
@@ -1415,40 +946,6 @@ handle_get_peer (void *cls,
 
 
 /**
- * Check that message received from CADET service is well-formed.
- *
- * @param cls the `struct GNUNET_CADET_Handle`
- * @param msg the message we got
- * @return #GNUNET_OK if the message is well-formed,
- *         #GNUNET_SYSERR otherwise
- */
-static int
-check_get_tunnels (void *cls,
-                   const struct GNUNET_CADET_LocalInfoTunnel *msg)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  uint16_t size;
-
-  if (NULL == h->info_cb.tunnels_cb)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "  no handler for tunnels monitor message!\n");
-    return GNUNET_SYSERR;
-  }
-
-  size = ntohs (msg->header.size);
-  if (sizeof (struct GNUNET_CADET_LocalInfoTunnel) > size)
-  {
-    h->info_cb.tunnels_cb (h->info_cls, NULL, 0, 0, 0, 0);
-    h->info_cb.tunnels_cb = NULL;
-    h->info_cls = NULL;
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
  * Process a local reply about info on all tunnels, pass info to the user.
  *
  * @param cls Closure (Cadet handle).
@@ -1460,6 +957,8 @@ handle_get_tunnels (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
 
+  if (NULL == h->info_cb.tunnels_cb)
+    return;
   h->info_cb.tunnels_cb (h->info_cls,
                          &msg->destination,
                          ntohl (msg->channels),
@@ -1482,28 +981,18 @@ static int
 check_get_tunnel (void *cls,
                   const struct GNUNET_CADET_LocalInfoTunnel *msg)
 {
-  struct GNUNET_CADET_Handle *h = cls;
   unsigned int ch_n;
   unsigned int c_n;
   size_t esize;
   size_t msize;
 
-  if (NULL == h->info_cb.tunnel_cb)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "  no handler for tunnel monitor message!\n");
-    goto clean_cls;
-  }
-
   /* Verify message sanity */
   msize = ntohs (msg->header.size);
   esize = sizeof (struct GNUNET_CADET_LocalInfoTunnel);
   if (esize > msize)
   {
-    GNUNET_break_op (0);
-    h->info_cb.tunnel_cb (h->info_cls,
-                          NULL, 0, 0, NULL, NULL, 0, 0);
-    goto clean_cls;
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
   ch_n = ntohl (msg->channels);
   c_n = ntohl (msg->connections);
@@ -1518,17 +1007,9 @@ check_get_tunnel (void *cls,
                 (unsigned int) esize,
                 ch_n,
                 c_n);
-    h->info_cb.tunnel_cb (h->info_cls,
-                          NULL, 0, 0, NULL, NULL, 0, 0);
-    goto clean_cls;
+    return GNUNET_SYSERR;
   }
-
   return GNUNET_OK;
-
-clean_cls:
-  h->info_cb.tunnel_cb = NULL;
-  h->info_cls = NULL;
-  return GNUNET_SYSERR;
 }
 
 
@@ -1548,6 +1029,9 @@ handle_get_tunnel (void *cls,
   const struct GNUNET_CADET_ConnectionTunnelIdentifier *conns;
   const struct GNUNET_CADET_ChannelTunnelNumber *chns;
 
+  if (NULL == h->info_cb.tunnel_cb)
+    return;
+
   ch_n = ntohl (msg->channels);
   c_n = ntohl (msg->connections);
 
@@ -1591,44 +1075,25 @@ reconnect (struct GNUNET_CADET_Handle *h)
                              GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
                              struct GNUNET_CADET_LocalAck,
                              h),
-    GNUNET_MQ_hd_var_size (get_peers,
-                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
-                           struct GNUNET_CADET_LocalInfoPeer,
-                           h),
+    GNUNET_MQ_hd_fixed_size (get_peers,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
+                             struct GNUNET_CADET_LocalInfoPeer,
+                             h),
     GNUNET_MQ_hd_var_size (get_peer,
                            GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
                            struct GNUNET_CADET_LocalInfoPeer,
                            h),
-    GNUNET_MQ_hd_var_size (get_tunnels,
-                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
-                           struct GNUNET_CADET_LocalInfoTunnel,
-                           h),
+    GNUNET_MQ_hd_fixed_size (get_tunnels,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
+                             struct GNUNET_CADET_LocalInfoTunnel,
+                             h),
     GNUNET_MQ_hd_var_size (get_tunnel,
                            GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
                            struct GNUNET_CADET_LocalInfoTunnel,
                            h),
-// FIXME
-//   GNUNET_MQ_hd_fixed_Y       size (channel_destroyed,
-//                        GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED,
-//                        struct GNUNET_CADET_ChannelDestroyMessage);
     GNUNET_MQ_handler_end ()
   };
-  struct GNUNET_CADET_Channel *ch;
-
-  while (NULL != (ch = h->channels_head))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Destroying channel due to a reconnect\n");
-    destroy_channel (ch);
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Connecting to CADET\n");
 
-  if (NULL != h->mq)
-  {
-    GNUNET_MQ_destroy (h->mq);
-    h->mq = NULL;
-  }
   h->mq = GNUNET_CLIENT_connect (h->cfg,
                                  "cadet",
                                  handlers,
@@ -1639,85 +1104,60 @@ reconnect (struct GNUNET_CADET_Handle *h)
     schedule_reconnect (h);
     return;
   }
-  else
-  {
-    h->reconnect_time = GNUNET_TIME_UNIT_MILLISECONDS;
-  }
+  h->reconnect_time = GNUNET_TIME_UNIT_MILLISECONDS;
 }
 
+
 /**
- * Reconnect callback: tries to reconnect again after a failer previous
- * reconnecttion
+ * Function called during #GNUNET_CADET_disconnect() to destroy
+ * all channels that are still open.
  *
- * @param cls closure (cadet handle)
+ * @param cls the `struct GNUNET_CADET_Handle`
+ * @param cid chanenl ID
+ * @param value a `struct GNUNET_CADET_Channel` to destroy
+ * @return #GNUNET_OK (continue to iterate)
  */
-static void
-reconnect_cbk (void *cls)
+static int
+destroy_channel_cb (void *cls,
+                    uint32_t cid,
+                    void *value)
 {
-  struct GNUNET_CADET_Handle *h = cls;
+  /* struct GNUNET_CADET_Handle *handle = cls; */
+  struct GNUNET_CADET_Channel *ch = value;
 
-  h->reconnect_task = NULL;
-  reconnect (h);
+  if (ntohl (ch->ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+  {
+    GNUNET_break (0);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "channel %X not destroyed\n",
+         ntohl (ch->ccn.channel_of_client));
+  }
+  destroy_channel (ch);
+  return GNUNET_OK;
 }
 
 
 /**
- * Reconnect to the service, retransmit all infomation to try to restore the
- * original state.
+ * Function called during #GNUNET_CADET_disconnect() to destroy
+ * all ports that are still open.
  *
- * @param h handle to the cadet
- *
- * @return #GNUNET_YES in case of sucess, #GNUNET_NO otherwise (service down...)
+ * @param cls the `struct GNUNET_CADET_Handle`
+ * @param id port ID
+ * @param value a `struct GNUNET_CADET_Channel` to destroy
+ * @return #GNUNET_OK (continue to iterate)
  */
-static void
-schedule_reconnect (struct GNUNET_CADET_Handle *h)
-{
-  if (NULL == h->reconnect_task)
-  {
-    h->reconnect_task = GNUNET_SCHEDULER_add_delayed (h->reconnect_time,
-                                                      &reconnect_cbk, h);
-    h->reconnect_time = GNUNET_TIME_STD_BACKOFF (h->reconnect_time);
-  }
-}
-
-
-/******************************************************************************/
-/**********************      API CALL DEFINITIONS     *************************/
-/******************************************************************************/
-
-struct GNUNET_CADET_Handle *
-GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      void *cls,
-                      GNUNET_CADET_ChannelEndHandler cleaner,
-                      const struct GNUNET_CADET_MessageHandler *handlers)
+static int
+destroy_port_cb (void *cls,
+                 const struct GNUNET_HashCode *id,
+                 void *value)
 {
-  struct GNUNET_CADET_Handle *h;
-
-  h = GNUNET_new (struct GNUNET_CADET_Handle);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "GNUNET_CADET_connect() %p\n",
-       h);
-  h->cfg = cfg;
-  h->cleaner = cleaner;
-  h->ports = GNUNET_CONTAINER_multihashmap_create (4, GNUNET_YES);
-  reconnect (h);
-  if (h->mq == NULL)
-  {
-    GNUNET_break (0);
-    GNUNET_CADET_disconnect (h);
-    return NULL;
-  }
-  h->cls = cls;
-  h->message_handlers = handlers;
-  h->next_ccn.channel_of_client = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-  h->reconnect_time = GNUNET_TIME_UNIT_MILLISECONDS;
-  h->reconnect_task = NULL;
+  /* struct GNUNET_CADET_Handle *handle = cls; */
+  struct GNUNET_CADET_Port *port = value;
 
-  /* count handlers */
-  for (h->n_handlers = 0;
-       handlers && handlers[h->n_handlers].type;
-       h->n_handlers++) ;
-  return h;
+  /* This is a warning, the app should have cleanly closed all open ports */
+  GNUNET_break (0);
+  GNUNET_CADET_close_port (port);
+  return GNUNET_OK;
 }
 
 
@@ -1732,57 +1172,16 @@ GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
 void
 GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle)
 {
-  struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_Channel *aux;
-  struct GNUNET_CADET_TransmitHandle *th;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "CADET DISCONNECT\n");
-  ch = handle->channels_head;
-  while (NULL != ch)
-  {
-    aux = ch->next;
-    if (ntohl (ch->ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-    {
-      GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "channel %X not destroyed\n",
-           ntohl (ch->ccn.channel_of_client));
-    }
-    destroy_channel (ch);
-    ch = aux;
-  }
-  while (NULL != (th = handle->th_head))
-  {
-    struct GNUNET_MessageHeader *msg;
-
-    /* Make sure it is an allowed packet (everything else should have been
-     * already canceled).
-     */
-    GNUNET_break (GNUNET_NO == th_is_payload (th));
-    msg = (struct GNUNET_MessageHeader *) &th[1];
-    switch (ntohs(msg->type))
-    {
-      case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-      case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNELS:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNEL:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL:
-      case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS:
-        break;
-      default:
-        GNUNET_break (0);
-        LOG (GNUNET_ERROR_TYPE_ERROR, "unexpected unsent msg %s\n",
-             GC_m2s (ntohs(msg->type)));
-    }
-
-    GNUNET_CADET_notify_transmit_ready_cancel (th);
-  }
-
+  GNUNET_CONTAINER_multihashmap_iterate (handle->ports,
+                                         &destroy_port_cb,
+                                         handle);
+  GNUNET_CONTAINER_multihashmap_destroy (handle->ports);
+  handle->ports = NULL;
+  GNUNET_CONTAINER_multihashmap32_iterate (handle->channels,
+                                           &destroy_channel_cb,
+                                           handle);
+  GNUNET_CONTAINER_multihashmap32_destroy (handle->channels);
+  handle->channels = NULL;
   if (NULL != handle->mq)
   {
     GNUNET_MQ_destroy (handle->mq);
@@ -1790,58 +1189,15 @@ GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle)
   }
   if (NULL != handle->reconnect_task)
   {
-    GNUNET_SCHEDULER_cancel(handle->reconnect_task);
+    GNUNET_SCHEDULER_cancel (handle->reconnect_task);
     handle->reconnect_task = NULL;
   }
-
-  GNUNET_CONTAINER_multihashmap_destroy (handle->ports);
-  handle->ports = NULL;
   GNUNET_free (handle);
 }
 
 
 /**
- * Open a port to receive incomming channels.
- *
- * @param h CADET handle.
- * @param port Hash representing the port number.
- * @param new_channel Function called when an channel is received.
- * @param new_channel_cls Closure for @a new_channel.
- * @return Port handle.
- */
-struct GNUNET_CADET_Port *
-GNUNET_CADET_open_port (struct GNUNET_CADET_Handle *h,
-                        const struct GNUNET_HashCode *port,
-                        GNUNET_CADET_InboundChannelNotificationHandler
-                            new_channel,
-                        void *new_channel_cls)
-{
-  struct GNUNET_CADET_PortMessage *msg;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_Port *p;
-
-  GNUNET_assert (NULL != new_channel);
-  p = GNUNET_new (struct GNUNET_CADET_Port);
-  p->cadet = h;
-  p->hash = GNUNET_new (struct GNUNET_HashCode);
-  *p->hash = *port;
-  p->handler = new_channel;
-  p->cls = new_channel_cls;
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multihashmap_put (h->ports,
-                                                    p->hash,
-                                                    p,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN);
-  msg->port = *p->hash;
-  GNUNET_MQ_send (h->mq, env);
-
-  return p;
-}
-
-/**
- * Close a port opened with @a GNUNET_CADET_open_port.
+ * Close a port opened with @a GNUNET_CADET_open_port().
  * The @a new_channel callback will no longer be called.
  *
  * @param p Port handle.
@@ -1851,111 +1207,45 @@ GNUNET_CADET_close_port (struct GNUNET_CADET_Port *p)
 {
   struct GNUNET_CADET_PortMessage *msg;
   struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_HashCode *id;
-
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE);
 
-  id = NULL != p->hash ? p->hash : &p->id;
-  msg->port = *id;
-  GNUNET_MQ_send (p->cadet->mq, env);
-  GNUNET_CONTAINER_multihashmap_remove (p->cadet->ports, id, p);
-  GNUNET_free_non_null (p->hash);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE);
+  msg->port = p->id;
+  GNUNET_MQ_send (p->cadet->mq,
+                  env);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (p->cadet->ports,
+                                                       &p->id,
+                                                       p));
+  GNUNET_free_non_null (p->handlers);
   GNUNET_free (p);
 }
 
 
 /**
- * Create a new channel towards a remote peer.
+ * Destroy an existing channel.
  *
- * If the destination port is not open by any peer or the destination peer
- * does not accept the channel, #GNUNET_CADET_ChannelEndHandler will be called
- * for this channel.
+ * The existing end callback for the channel will be called immediately.
+ * Any pending outgoing messages will be sent but no incoming messages will be
+ * accepted and no data callbacks will be called.
  *
- * @param h cadet handle
- * @param channel_ctx client's channel context to associate with the channel
- * @param peer peer identity the channel should go to
- * @param port Port hash (port number).
- * @param options CadetOption flag field, with all desired option bits set to 1.
- * @return handle to the channel
+ * @param channel Channel handle, becomes invalid after this call.
  */
-struct GNUNET_CADET_Channel *
-GNUNET_CADET_channel_create (struct GNUNET_CADET_Handle *h,
-                            void *channel_ctx,
-                            const struct GNUNET_PeerIdentity *peer,
-                            const struct GNUNET_HashCode *port,
-                            enum GNUNET_CADET_ChannelOption options)
-{
-  struct GNUNET_CADET_LocalChannelCreateMessage *msg;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  ccn.channel_of_client = htonl (0);
-  ch = create_channel (h, ccn);
-  ch->ctx = channel_ctx;
-  ch->peer = GNUNET_PEER_intern (peer);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating new channel to %s:%u at %p number %X\n",
-       GNUNET_i2s (peer),
-       port,
-       ch,
-       ntohl (ch->ccn.channel_of_client));
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
-  msg->ccn = ch->ccn;
-  msg->port = *port;
-  msg->peer = *peer;
-  msg->opt = htonl (options);
-  GNUNET_MQ_send (h->mq,
-                  env);
-  return ch;
-}
-
-
 void
 GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel)
 {
-  struct GNUNET_CADET_Handle *h;
+  struct GNUNET_CADET_Handle *h = channel->cadet;
   struct GNUNET_CADET_LocalChannelDestroyMessage *msg;
   struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_TransmitHandle *th;
-  struct GNUNET_CADET_TransmitHandle *next;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying channel\n");
-  h = channel->cadet;
-  for  (th = h->th_head; th != NULL; th = next)
+  if (NULL != h->mq)
   {
-    next = th->next;
-    if (th->channel == channel)
-    {
-      GNUNET_break (0);
-      if (GNUNET_YES == th_is_payload (th))
-      {
-        /* applications should cancel before destroying channel */
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             "Channel destroyed without cancelling transmission requests\n");
-        th->notify (th->notify_cls, 0, NULL);
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             "no meta-traffic should be queued\n");
-      }
-      GNUNET_CONTAINER_DLL_remove (h->th_head,
-                                   h->th_tail,
-                                   th);
-      GNUNET_CADET_notify_transmit_ready_cancel (th);
-    }
+    env = GNUNET_MQ_msg (msg,
+                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
+    msg->ccn = channel->ccn;
+    GNUNET_MQ_send (h->mq,
+                    env);
   }
-
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
-  msg->ccn = channel->ccn;
-  GNUNET_MQ_send (h->mq,
-                  env);
-
   destroy_channel (channel);
 }
 
@@ -1971,10 +1261,10 @@ GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel)
  */
 const union GNUNET_CADET_ChannelInfo *
 GNUNET_CADET_channel_get_info (struct GNUNET_CADET_Channel *channel,
-                              enum GNUNET_CADET_ChannelOption option, ...)
+                               enum GNUNET_CADET_ChannelOption option,
+                               ...)
 {
   static int bool_flag;
-  const union GNUNET_CADET_ChannelInfo *ret;
 
   switch (option)
   {
@@ -1985,74 +1275,15 @@ GNUNET_CADET_channel_get_info (struct GNUNET_CADET_Channel *channel,
         bool_flag = GNUNET_YES;
       else
         bool_flag = GNUNET_NO;
-      ret = (const union GNUNET_CADET_ChannelInfo *) &bool_flag;
+      return (const union GNUNET_CADET_ChannelInfo *) &bool_flag;
       break;
     case GNUNET_CADET_OPTION_PEER:
-      ret = (const union GNUNET_CADET_ChannelInfo *) GNUNET_PEER_resolve2 (channel->peer);
+      return (const union GNUNET_CADET_ChannelInfo *) &channel->peer;
       break;
     default:
       GNUNET_break (0);
       return NULL;
   }
-
-  return ret;
-}
-
-
-struct GNUNET_CADET_TransmitHandle *
-GNUNET_CADET_notify_transmit_ready (struct GNUNET_CADET_Channel *channel,
-                                    int cork,
-                                    struct GNUNET_TIME_Relative maxdelay,
-                                    size_t notify_size,
-                                    GNUNET_CONNECTION_TransmitReadyNotify notify,
-                                    void *notify_cls)
-{
-  struct GNUNET_CADET_TransmitHandle *th;
-
-  GNUNET_assert (NULL != channel);
-  GNUNET_assert (NULL != notify);
-  GNUNET_assert (GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE >= notify_size);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "CADET NOTIFY TRANSMIT READY on channel %X allow_send is %u to %s with %u bytes\n",
-       ntohl (channel->ccn.channel_of_client),
-       channel->allow_send,
-       (ntohl (channel->ccn.channel_of_client) >=
-        GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-       ? "origin"
-       : "destination",
-       (unsigned int) notify_size);
-  if (GNUNET_TIME_UNIT_FOREVER_REL.rel_value_us != maxdelay.rel_value_us)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "CADET transmit ready timeout is deprected (has no effect)\n");
-  }
-
-  th = GNUNET_new (struct GNUNET_CADET_TransmitHandle);
-  th->channel = channel;
-  th->size = notify_size;
-  th->notify = notify;
-  th->notify_cls = notify_cls;
-  if (0 != channel->allow_send)
-    th->request_data_task
-      = GNUNET_SCHEDULER_add_now (&request_data,
-                                  th);
-  else
-    add_to_queue (channel->cadet,
-                  th);
-  return th;
-}
-
-
-void
-GNUNET_CADET_notify_transmit_ready_cancel (struct GNUNET_CADET_TransmitHandle *th)
-{
-  if (NULL != th->request_data_task)
-  {
-    GNUNET_SCHEDULER_cancel (th->request_data_task);
-    th->request_data_task = NULL;
-  }
-  remove_from_queue (th);
-  GNUNET_free (th);
 }
 
 
@@ -2078,18 +1309,23 @@ GNUNET_CADET_receive_done (struct GNUNET_CADET_Channel *channel)
 }
 
 
+/**
+ * Send message of @a type to CADET service of @a h
+ *
+ * @param h handle to CADET service
+ * @param type message type of trivial information request to send
+ */
 static void
-send_info_request (struct GNUNET_CADET_Handle *h, uint16_t type)
+send_info_request (struct GNUNET_CADET_Handle *h,
+                   uint16_t type)
 {
   struct GNUNET_MessageHeader *msg;
   struct GNUNET_MQ_Envelope *env;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " Sending %s monitor message to service\n",
-       GC_m2s(type));
-
-  env = GNUNET_MQ_msg (msg, type);
-  GNUNET_MQ_send (h->mq, env);
+  env = GNUNET_MQ_msg (msg,
+                       type);
+  GNUNET_MQ_send (h->mq,
+                  env);
 }
 
 
@@ -2103,8 +1339,8 @@ send_info_request (struct GNUNET_CADET_Handle *h, uint16_t type)
 void
 GNUNET_CADET_request_dump (struct GNUNET_CADET_Handle *h)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "requesting dump\n");
-  send_info_request (h, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP);
+  send_info_request (h,
+                     GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP);
 }
 
 
@@ -2113,13 +1349,11 @@ GNUNET_CADET_request_dump (struct GNUNET_CADET_Handle *h)
  * The callback will be called for every peer known to the service.
  * Only one info request (of any kind) can be active at once.
  *
- *
  * WARNING: unstable API, likely to change in the future!
  *
  * @param h Handle to the cadet peer.
  * @param callback Function to call with the requested data.
  * @param callback_cls Closure for @c callback.
- *
  * @return #GNUNET_OK / #GNUNET_SYSERR
  */
 int
@@ -2132,7 +1366,8 @@ GNUNET_CADET_get_peers (struct GNUNET_CADET_Handle *h,
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-  send_info_request (h, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
+  send_info_request (h,
+                     GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
   h->info_cb.peers_cb = callback;
   h->info_cls = callback_cls;
   return GNUNET_OK;
@@ -2145,15 +1380,13 @@ GNUNET_CADET_get_peers (struct GNUNET_CADET_Handle *h,
  * WARNING: unstable API, likely to change in the future!
  *
  * @param h Cadet handle.
- *
- * @return Closure given to GNUNET_CADET_get_peers.
+ * @return Closure given to GNUNET_CADET_get_peers().
  */
 void *
 GNUNET_CADET_get_peers_cancel (struct GNUNET_CADET_Handle *h)
 {
-  void *cls;
+  void *cls = h->info_cls;
 
-  cls = h->info_cls;
   h->info_cb.peers_cb = NULL;
   h->info_cls = NULL;
   return cls;
@@ -2171,7 +1404,6 @@ GNUNET_CADET_get_peers_cancel (struct GNUNET_CADET_Handle *h)
  * @param id Peer whose tunnel to examine.
  * @param callback Function to call with the requested data.
  * @param callback_cls Closure for @c callback.
- *
  * @return #GNUNET_OK / #GNUNET_SYSERR
  */
 int
@@ -2188,11 +1420,11 @@ GNUNET_CADET_get_peer (struct GNUNET_CADET_Handle *h,
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
   msg->peer = *id;
-  GNUNET_MQ_send (h->mq, env);
-
+  GNUNET_MQ_send (h->mq,
+                  env);
   h->info_cb.peer_cb = callback;
   h->info_cls = callback_cls;
   return GNUNET_OK;
@@ -2209,7 +1441,6 @@ GNUNET_CADET_get_peer (struct GNUNET_CADET_Handle *h,
  * @param h Handle to the cadet peer.
  * @param callback Function to call with the requested data.
  * @param callback_cls Closure for @c callback.
- *
  * @return #GNUNET_OK / #GNUNET_SYSERR
  */
 int
@@ -2222,7 +1453,8 @@ GNUNET_CADET_get_tunnels (struct GNUNET_CADET_Handle *h,
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-  send_info_request (h, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
+  send_info_request (h,
+                     GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
   h->info_cb.tunnels_cb = callback;
   h->info_cls = callback_cls;
   return GNUNET_OK;
@@ -2233,23 +1465,19 @@ GNUNET_CADET_get_tunnels (struct GNUNET_CADET_Handle *h,
  * Cancel a monitor request. The monitor callback will not be called.
  *
  * @param h Cadet handle.
- *
- * @return Closure given to GNUNET_CADET_get_tunnels.
+ * @return Closure given to GNUNET_CADET_get_tunnels().
  */
 void *
 GNUNET_CADET_get_tunnels_cancel (struct GNUNET_CADET_Handle *h)
 {
-  void *cls;
+  void *cls = h->info_cls;
 
   h->info_cb.tunnels_cb = NULL;
-  cls = h->info_cls;
   h->info_cls = NULL;
-
   return cls;
 }
 
 
-
 /**
  * Request information about a tunnel of the running cadet peer.
  * The callback will be called for the tunnel once.
@@ -2261,7 +1489,6 @@ GNUNET_CADET_get_tunnels_cancel (struct GNUNET_CADET_Handle *h)
  * @param id Peer whose tunnel to examine.
  * @param callback Function to call with the requested data.
  * @param callback_cls Closure for @c callback.
- *
  * @return #GNUNET_OK / #GNUNET_SYSERR
  */
 int
@@ -2278,11 +1505,11 @@ GNUNET_CADET_get_tunnel (struct GNUNET_CADET_Handle *h,
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
   msg->peer = *id;
-  GNUNET_MQ_send (h->mq, env);
-
+  GNUNET_MQ_send (h->mq,
+                  env);
   h->info_cb.tunnel_cb = callback;
   h->info_cls = callback_cls;
   return GNUNET_OK;
@@ -2290,158 +1517,6 @@ GNUNET_CADET_get_tunnel (struct GNUNET_CADET_Handle *h,
 
 
 /**
- * Request information about a specific channel of the running cadet peer.
- *
- * WARNING: unstable API, likely to change in the future!
- * FIXME Add destination option.
- *
- * @param h Handle to the cadet peer.
- * @param initiator ID of the owner of the channel.
- * @param channel_number Channel number.
- * @param callback Function to call with the requested data.
- * @param callback_cls Closure for @c callback.
- *
- * @return #GNUNET_OK / #GNUNET_SYSERR
- */
-int
-GNUNET_CADET_show_channel (struct GNUNET_CADET_Handle *h,
-                           struct GNUNET_PeerIdentity *initiator,
-                           unsigned int channel_number,
-                           GNUNET_CADET_ChannelCB callback,
-                           void *callback_cls)
-{
-  struct GNUNET_CADET_LocalInfo *msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  if (NULL != h->info_cb.channel_cb)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNEL);
-  msg->peer = *initiator;
-  msg->ccn.channel_of_client = htonl (channel_number);
-  GNUNET_MQ_send (h->mq, env);
-
-  h->info_cb.channel_cb = callback;
-  h->info_cls = callback_cls;
-  return GNUNET_OK;
-}
-
-
-/**
- * Function called to notify a client about the connection
- * begin ready to queue more data.  "buf" will be
- * NULL and "size" zero if the connection was closed for
- * writing in the meantime.
- *
- * @param cls closure
- * @param size number of bytes available in buf
- * @param buf where the callee should write the message
- * @return number of bytes written to buf
- */
-static size_t
-cadet_mq_ntr (void *cls, size_t size,
-             void *buf)
-{
-  struct GNUNET_MQ_Handle *mq = cls;
-  struct CadetMQState *state = GNUNET_MQ_impl_state (mq);
-  const struct GNUNET_MessageHeader *msg = GNUNET_MQ_impl_current (mq);
-  uint16_t msize;
-
-  state->th = NULL;
-  if (NULL == buf)
-  {
-    GNUNET_MQ_inject_error (mq, GNUNET_MQ_ERROR_WRITE);
-    return 0;
-  }
-  msize = ntohs (msg->size);
-  GNUNET_assert (msize <= size);
-  GNUNET_memcpy (buf, msg, msize);
-  GNUNET_MQ_impl_send_continue (mq);
-  return msize;
-}
-
-
-/**
- * Signature of functions implementing the
- * sending functionality of a message queue.
- *
- * @param mq the message queue
- * @param msg the message to send
- * @param impl_state state of the implementation
- */
-static void
-cadet_mq_send_impl_old (struct GNUNET_MQ_Handle *mq,
-                        const struct GNUNET_MessageHeader *msg,
-                        void *impl_state)
-{
-  struct CadetMQState *state = impl_state;
-
-  GNUNET_assert (NULL == state->th);
-  state->th =
-      GNUNET_CADET_notify_transmit_ready (state->channel,
-                                         /* FIXME: add option for corking */
-                                         GNUNET_NO,
-                                         GNUNET_TIME_UNIT_FOREVER_REL,
-                                         ntohs (msg->size),
-                                         &cadet_mq_ntr, mq);
-
-}
-
-
-/**
- * Signature of functions implementing the
- * destruction of a message queue.
- * Implementations must not free 'mq', but should
- * take care of 'impl_state'.
- *
- * @param mq the message queue to destroy
- * @param impl_state state of the implementation
- */
-static void
-cadet_mq_destroy_impl_old (struct GNUNET_MQ_Handle *mq,
-                           void *impl_state)
-{
-  struct CadetMQState *state = impl_state;
-
-  if (NULL != state->th)
-    GNUNET_CADET_notify_transmit_ready_cancel (state->th);
-
-  GNUNET_free (state);
-}
-
-
-/**
- * Create a message queue for a cadet channel.
- * The message queue can only be used to transmit messages,
- * not to receive them.
- *
- * @param channel the channel to create the message qeue for
- * @return a message queue to messages over the channel
- */
-struct GNUNET_MQ_Handle *
-GNUNET_CADET_mq_create (struct GNUNET_CADET_Channel *channel)
-{
-  struct GNUNET_MQ_Handle *mq;
-  struct CadetMQState *state;
-
-  state = GNUNET_new (struct CadetMQState);
-  state->channel = channel;
-
-  mq = GNUNET_MQ_queue_for_callbacks (&cadet_mq_send_impl_old,
-                                      &cadet_mq_destroy_impl_old,
-                                      NULL, /* FIXME: cancel impl. */
-                                      state,
-                                      NULL, /* no msg handlers */
-                                      NULL, /* no err handlers */
-                                      NULL); /* no handler cls */
-  return mq;
-}
-
-
-/**
  * Transitional function to convert an unsigned int port to a hash value.
  * WARNING: local static value returned, NOT reentrant!
  * WARNING: do not use this function for new code!
@@ -2456,19 +1531,14 @@ GC_u2h (uint32_t port)
   static struct GNUNET_HashCode hash;
 
   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-              "This is a transitional function, "
-              "use proper crypto hashes as CADET ports\n");
-  GNUNET_CRYPTO_hash (&port, sizeof (port), &hash);
-
+              "This is a transitional function, use proper crypto hashes as CADET ports\n");
+  GNUNET_CRYPTO_hash (&port,
+                      sizeof (port),
+                      &hash);
   return &hash;
 }
 
 
-
-/******************************************************************************/
-/******************************* MQ-BASED API *********************************/
-/******************************************************************************/
-
 /**
  * Connect to the MQ-based cadet service.
  *
@@ -2477,16 +1547,17 @@ GC_u2h (uint32_t port)
  * @return Handle to the cadet service NULL on error.
  */
 struct GNUNET_CADET_Handle *
-GNUNET_CADET_connecT (const struct GNUNET_CONFIGURATION_Handle *cfg)
+GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_CADET_Handle *h;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "GNUNET_CADET_connecT()\n");
+       "GNUNET_CADET_connect()\n");
   h = GNUNET_new (struct GNUNET_CADET_Handle);
   h->cfg = cfg;
-  h->mq_api = GNUNET_YES;
-  h->ports = GNUNET_CONTAINER_multihashmap_create (4, GNUNET_YES);
+  h->ports = GNUNET_CONTAINER_multihashmap_create (4,
+                                                   GNUNET_YES);
+  h->channels = GNUNET_CONTAINER_multihashmap32_create (4);
   reconnect (h);
   if (NULL == h->mq)
   {
@@ -2512,11 +1583,10 @@ GNUNET_CADET_connecT (const struct GNUNET_CONFIGURATION_Handle *cfg)
  * @param window_changes Function called when the transmit window size changes.
  * @param disconnects Function called when a channel is disconnected.
  * @param handlers Callbacks for messages we care about, NULL-terminated.
- *
  * @return Port handle.
  */
 struct GNUNET_CADET_Port *
-GNUNET_CADET_open_porT (struct GNUNET_CADET_Handle *h,
+GNUNET_CADET_open_port (struct GNUNET_CADET_Handle *h,
                         const struct GNUNET_HashCode *port,
                         GNUNET_CADET_ConnectEventHandler connects,
                         void * connects_cls,
@@ -2538,16 +1608,7 @@ GNUNET_CADET_open_porT (struct GNUNET_CADET_Handle *h,
   p->cls = connects_cls;
   p->window_changes = window_changes;
   p->disconnects = disconnects;
-  if (NULL != handlers)
-  {
-    unsigned int i;
-    for (i=0;NULL != handlers[i].cb; i++) ;
-    p->handlers = GNUNET_new_array (i + 1,
-                                    struct GNUNET_MQ_MessageHandler);
-    GNUNET_memcpy ((struct GNUNET_MQ_MessageHandler *) p->handlers,
-                   handlers,
-                   i * sizeof (struct GNUNET_MQ_MessageHandler));
-  }
+  p->handlers = GNUNET_MQ_copy_handlers (handlers);
 
   GNUNET_assert (GNUNET_OK ==
                  GNUNET_CONTAINER_multihashmap_put (h->ports,
@@ -2555,10 +1616,11 @@ GNUNET_CADET_open_porT (struct GNUNET_CADET_Handle *h,
                                                     p,
                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN);
   msg->port = p->id;
-  GNUNET_MQ_send (h->mq, env);
-
+  GNUNET_MQ_send (h->mq,
+                  env);
   return p;
 }
 
@@ -2580,11 +1642,10 @@ GNUNET_CADET_open_porT (struct GNUNET_CADET_Handle *h,
  * @param window_changes Function called when the transmit window size changes.
  * @param disconnects Function called when the channel is disconnected.
  * @param handlers Callbacks for messages we care about, NULL-terminated.
- *
  * @return Handle to the channel.
  */
 struct GNUNET_CADET_Channel *
-GNUNET_CADET_channel_creatE (struct GNUNET_CADET_Handle *h,
+GNUNET_CADET_channel_create (struct GNUNET_CADET_Handle *h,
                              void *channel_cls,
                              const struct GNUNET_PeerIdentity *destination,
                              const struct GNUNET_HashCode *port,
@@ -2594,17 +1655,14 @@ GNUNET_CADET_channel_creatE (struct GNUNET_CADET_Handle *h,
                              const struct GNUNET_MQ_MessageHandler *handlers)
 {
   struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
   struct GNUNET_CADET_LocalChannelCreateMessage *msg;
   struct GNUNET_MQ_Envelope *env;
 
   GNUNET_assert (NULL != disconnects);
-
-  /* Save parameters */
-  ccn.channel_of_client = htonl (0);
-  ch = create_channel (h, ccn);
+  ch = create_channel (h,
+                       NULL);
   ch->ctx = channel_cls;
-  ch->peer = GNUNET_PEER_intern (destination);
+  ch->peer = *destination;
   ch->options = options;
   ch->window_changes = window_changes;
   ch->disconnects = disconnects;
@@ -2628,7 +1686,6 @@ GNUNET_CADET_channel_creatE (struct GNUNET_CADET_Handle *h,
   msg->opt = htonl (options);
   GNUNET_MQ_send (h->mq,
                   env);
-
   return ch;
 }
 
@@ -2645,3 +1702,5 @@ GNUNET_CADET_get_mq (const struct GNUNET_CADET_Channel *channel)
 {
   return channel->mq;
 }
+
+/* end of cadet_api.c */
diff --git a/src/cadet/cadet_api_new.c b/src/cadet/cadet_api_new.c
deleted file mode 100644
index a62de0a47..000000000
--- a/src/cadet/cadet_api_new.c
+++ /dev/null
@@ -1,1720 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/cadet_api.c
- * @brief cadet api: client implementation of cadet service
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_constants.h"
-#include "gnunet_cadet_service.h"
-#include "cadet.h"
-#include "cadet_protocol.h"
-
-#define LOG(kind,...) GNUNET_log_from (kind, "cadet-api",__VA_ARGS__)
-
-/**
- * Ugly legacy hack.
- */
-union CadetInfoCB
-{
-
-  /**
-   * Channel callback.
-   */
-  GNUNET_CADET_ChannelCB channel_cb;
-
-  /**
-   * Monitor callback
-   */
-  GNUNET_CADET_PeersCB peers_cb;
-
-  /**
-   * Monitor callback
-   */
-  GNUNET_CADET_PeerCB peer_cb;
-
-  /**
-   * Monitor callback
-   */
-  GNUNET_CADET_TunnelsCB tunnels_cb;
-
-  /**
-   * Tunnel callback.
-   */
-  GNUNET_CADET_TunnelCB tunnel_cb;
-};
-
-
-/**
- * Opaque handle to the service.
- */
-struct GNUNET_CADET_Handle
-{
-  /**
-   * Message queue.
-   */
-  struct GNUNET_MQ_Handle *mq;
-
-  /**
-   * Ports open.
-   */
-  struct GNUNET_CONTAINER_MultiHashMap *ports;
-
-  /**
-   * Channels open.
-   */
-  struct GNUNET_CONTAINER_MultiHashMap32 *channels;
-
-  /**
-   * child of the next channel to create (to avoid reusing IDs often)
-   */
-  struct GNUNET_CADET_ClientChannelNumber next_ccn;
-
-  /**
-   * Configuration given by the client, in case of reconnection
-   */
-  const struct GNUNET_CONFIGURATION_Handle *cfg;
-
-  /**
-   * Task for trying to reconnect.
-   */
-  struct GNUNET_SCHEDULER_Task *reconnect_task;
-
-  /**
-   * Callback for an info task (only one active at a time).
-   */
-  union CadetInfoCB info_cb;
-
-  /**
-   * Info callback closure for @c info_cb.
-   */
-  void *info_cls;
-
-  /**
-   * Time to the next reconnect in case one reconnect fails
-   */
-  struct GNUNET_TIME_Relative reconnect_time;
-
-};
-
-
-/**
- * Opaque handle to a channel.
- */
-struct GNUNET_CADET_Channel
-{
-
-  /**
-   * Other end of the channel.
-   */
-  struct GNUNET_PeerIdentity peer;
-
-  /**
-   * Handle to the cadet this channel belongs to
-   */
-  struct GNUNET_CADET_Handle *cadet;
-
-  /**
-   * Channel's port, if incoming.
-   */
-  struct GNUNET_CADET_Port *incoming_port;
-
-  /**
-   * Any data the caller wants to put in here, used for the
-   * various callbacks (@e disconnects, @e window_changes, handlers).
-   */
-  void *ctx;
-
-  /**
-   * Message Queue for the channel (which we are implementing).
-   */
-  struct GNUNET_MQ_Handle *mq;
-
-  /**
-   * Task to allow mq to send more traffic.
-   */
-  struct GNUNET_SCHEDULER_Task *mq_cont;
-
-  /**
-   * Pending envelope with a message to be transmitted to the
-   * service as soon as we are allowed to.  Should only be
-   * non-NULL if @e allow_send is 0.
-   */
-  struct GNUNET_MQ_Envelope *pending_env;
-
-  /**
-   * Window change handler.
-   */
-  GNUNET_CADET_WindowSizeEventHandler window_changes;
-
-  /**
-   * Disconnect handler.
-   */
-  GNUNET_CADET_DisconnectEventHandler disconnects;
-
-  /**
-   * Local ID of the channel, #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI bit is set if outbound.
-   */
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  /**
-   * Channel options: reliability, etc.
-   */
-  enum GNUNET_CADET_ChannelOption options;
-
-  /**
-   * How many messages are we allowed to send to the service right now?
-   */
-  unsigned int allow_send;
-
-};
-
-
-/**
- * Opaque handle to a port.
- */
-struct GNUNET_CADET_Port
-{
-
-  /**
-   * Port "number"
-   */
-  struct GNUNET_HashCode id;
-
-  /**
-   * Handle to the CADET session this port belongs to.
-   */
-  struct GNUNET_CADET_Handle *cadet;
-
-  /**
-   * Callback handler for incoming channels on this port.
-   */
-  GNUNET_CADET_InboundChannelNotificationHandler *handler;
-
-  /**
-   * Closure for @a handler.
-   */
-  void *cls;
-
-  /**
-   * Handler for incoming channels on this port
-   */
-  GNUNET_CADET_ConnectEventHandler connects;
-
-  /**
-   * Closure for @ref connects
-   */
-  void *connects_cls;
-
-  /**
-   * Window size change handler.
-   */
-  GNUNET_CADET_WindowSizeEventHandler window_changes;
-
-  /**
-   * Handler called when an incoming channel is destroyed.
-   */
-  GNUNET_CADET_DisconnectEventHandler disconnects;
-
-  /**
-   * Payload handlers for incoming channels.
-   */
-  struct GNUNET_MQ_MessageHandler *handlers;
-};
-
-
-/**
- * Find the Port struct for a hash.
- *
- * @param h CADET handle.
- * @param hash HashCode for the port number.
- * @return The port handle if known, NULL otherwise.
- */
-static struct GNUNET_CADET_Port *
-find_port (const struct GNUNET_CADET_Handle *h,
-           const struct GNUNET_HashCode *hash)
-{
-  return GNUNET_CONTAINER_multihashmap_get (h->ports,
-                                            hash);
-}
-
-
-/**
- * Get the channel handler for the channel specified by id from the given handle
- *
- * @param h Cadet handle
- * @param ccn ID of the wanted channel
- * @return handle to the required channel or NULL if not found
- */
-static struct GNUNET_CADET_Channel *
-find_channel (struct GNUNET_CADET_Handle *h,
-              struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  return GNUNET_CONTAINER_multihashmap32_get (h->channels,
-                                              ntohl (ccn.channel_of_client));
-}
-
-
-/**
- * Create a new channel and insert it in the channel list of the cadet handle
- *
- * @param h Cadet handle
- * @param ccnp pointer to desired ccn of the channel, NULL to assign one automatically.
- * @return Handle to the created channel.
- */
-static struct GNUNET_CADET_Channel *
-create_channel (struct GNUNET_CADET_Handle *h,
-                const struct GNUNET_CADET_ClientChannelNumber *ccnp)
-{
-  struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  ch = GNUNET_new (struct GNUNET_CADET_Channel);
-  ch->cadet = h;
-  if (NULL == ccnp)
-  {
-    while (NULL !=
-           find_channel (h,
-                         h->next_ccn))
-      h->next_ccn.channel_of_client
-        = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI | (1 + ntohl (h->next_ccn.channel_of_client)));
-    ccn = h->next_ccn;
-  }
-  else
-  {
-    ccn = *ccnp;
-  }
-  ch->ccn = ccn;
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multihashmap32_put (h->channels,
-                                                      ntohl (ch->ccn.channel_of_client),
-                                                      ch,
-                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  return ch;
-}
-
-
-/**
- * Destroy the specified channel.
- * - Destroys all peers, calling the disconnect callback on each if needed
- * - Cancels all outgoing traffic for that channel, calling respective notifys
- * - Calls cleaner if channel was inbound
- * - Frees all memory used
- *
- * @param ch Pointer to the channel.
- * @param call_cleaner Whether to call the cleaner handler.
- */
-static void
-destroy_channel (struct GNUNET_CADET_Channel *ch)
-{
-  struct GNUNET_CADET_Handle *h = ch->cadet;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying channel %X of %p\n",
-       ch->ccn,
-       h);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_remove (h->channels,
-                                                         ntohl (ch->ccn.channel_of_client),
-                                                         ch));
-  if (NULL != ch->mq_cont)
-  {
-    GNUNET_SCHEDULER_cancel (ch->mq_cont);
-    ch->mq_cont = NULL;
-  }
-  /* signal channel destruction */
-  if (NULL != ch->disconnects)
-    ch->disconnects (ch->ctx,
-                     ch);
-  if (NULL != ch->pending_env)
-    GNUNET_MQ_discard (ch->pending_env);
-  GNUNET_MQ_destroy (ch->mq);
-  GNUNET_free (ch);
-}
-
-
-/**
- * Reconnect to the service, retransmit all infomation to try to restore the
- * original state.
- *
- * @param h handle to the cadet
- */
-static void
-reconnect (struct GNUNET_CADET_Handle *h);
-
-
-/**
- * Reconnect callback: tries to reconnect again after a failer previous
- * reconnecttion
- *
- * @param cls closure (cadet handle)
- */
-static void
-reconnect_cbk (void *cls)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-
-  h->reconnect_task = NULL;
-  reconnect (h);
-}
-
-
-/**
- * Function called during #reconnect() to destroy
- * all channels that are still open.
- *
- * @param cls the `struct GNUNET_CADET_Handle`
- * @param cid chanenl ID
- * @param value a `struct GNUNET_CADET_Channel` to destroy
- * @return #GNUNET_OK (continue to iterate)
- */
-static int
-destroy_channel_on_reconnect_cb (void *cls,
-                                 uint32_t cid,
-                                 void *value)
-{
-  /* struct GNUNET_CADET_Handle *handle = cls; */
-  struct GNUNET_CADET_Channel *ch = value;
-
-  destroy_channel (ch);
-  return GNUNET_OK;
-}
-
-
-/**
- * Reconnect to the service, retransmit all infomation to try to restore the
- * original state.
- *
- * @param h handle to the cadet
- *
- * @return #GNUNET_YES in case of sucess, #GNUNET_NO otherwise (service down...)
- */
-static void
-schedule_reconnect (struct GNUNET_CADET_Handle *h)
-{
-  if (NULL != h->reconnect_task)
-    return;
-  GNUNET_CONTAINER_multihashmap32_iterate (h->channels,
-                                           &destroy_channel_on_reconnect_cb,
-                                           h);
-  h->reconnect_task
-    = GNUNET_SCHEDULER_add_delayed (h->reconnect_time,
-                                    &reconnect_cbk,
-                                    h);
-  h->reconnect_time
-    = GNUNET_TIME_STD_BACKOFF (h->reconnect_time);
-}
-
-
-/**
- * Notify the application about a change in the window size (if needed).
- *
- * @param ch Channel to notify about.
- */
-static void
-notify_window_size (struct GNUNET_CADET_Channel *ch)
-{
-  if (NULL != ch->window_changes)
-    ch->window_changes (ch->ctx,
-                        ch, /* FIXME: remove 'ch'? */
-                        ch->allow_send);
-}
-
-
-/**
- * Transmit the next message from our queue.
- *
- * @param cls Closure (channel whose mq to activate).
- */
-static void
-cadet_mq_send_now (void *cls)
-{
-  struct GNUNET_CADET_Channel *ch = cls;
-  struct GNUNET_MQ_Envelope *env = ch->pending_env;
-
-  ch->mq_cont = NULL;
-  if (0 == ch->allow_send)
-  {
-    /* how did we get here? */
-    GNUNET_break (0);
-    return;
-  }
-  if (NULL == env)
-  {
-    /* how did we get here? */
-    GNUNET_break (0);
-    return;
-  }
-  ch->allow_send--;
-  ch->pending_env = NULL;
-  GNUNET_MQ_send (ch->cadet->mq,
-                  env);
-  GNUNET_MQ_impl_send_continue (ch->mq);
-}
-
-
-/**
- * Implement sending functionality of a message queue for
- * us sending messages to a peer.
- *
- * Encapsulates the payload message in a #GNUNET_CADET_LocalData message
- * in order to label the message with the channel ID and send the
- * encapsulated message to the service.
- *
- * @param mq the message queue
- * @param msg the message to send
- * @param impl_state state of the implementation
- */
-static void
-cadet_mq_send_impl (struct GNUNET_MQ_Handle *mq,
-                    const struct GNUNET_MessageHeader *msg,
-                    void *impl_state)
-{
-  struct GNUNET_CADET_Channel *ch = impl_state;
-  struct GNUNET_CADET_Handle *h = ch->cadet;
-  uint16_t msize;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalData *cadet_msg;
-
-  if (NULL == h->mq)
-  {
-    /* We're currently reconnecting, pretend this worked */
-    GNUNET_MQ_impl_send_continue (mq);
-    return;
-  }
-
-  /* check message size for sanity */
-  msize = ntohs (msg->size);
-  if (msize > GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE)
-  {
-    GNUNET_break (0);
-    GNUNET_MQ_impl_send_continue (mq);
-    return;
-  }
-  env = GNUNET_MQ_msg_nested_mh (cadet_msg,
-                                 GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
-                                 msg);
-  cadet_msg->ccn = ch->ccn;
-  GNUNET_assert (NULL == ch->pending_env);
-  ch->pending_env = env;
-  if (0 < ch->allow_send)
-    ch->mq_cont
-      = GNUNET_SCHEDULER_add_now (&cadet_mq_send_now,
-                                  ch);
-}
-
-
-/**
- * Handle destruction of a message queue.  Implementations must not
- * free @a mq, but should take care of @a impl_state.
- *
- * @param mq the message queue to destroy
- * @param impl_state state of the implementation
- */
-static void
-cadet_mq_destroy_impl (struct GNUNET_MQ_Handle *mq,
-                       void *impl_state)
-{
-  struct GNUNET_CADET_Channel *ch = impl_state;
-
-  GNUNET_assert (mq == ch->mq);
-  ch->mq = NULL;
-}
-
-
-/**
- * We had an error processing a message we forwarded from a peer to
- * the CADET service.  We should just complain about it but otherwise
- * continue processing.
- *
- * @param cls closure with our `struct GNUNET_CADET_Channel`
- * @param error error code
- */
-static void
-cadet_mq_error_handler (void *cls,
-                        enum GNUNET_MQ_Error error)
-{
-  struct GNUNET_CADET_Channel *ch = cls;
-
-  GNUNET_break (0);
-  if (GNUNET_MQ_ERROR_NO_MATCH == error)
-  {
-    /* Got a message we did not understand, still try to continue! */
-    GNUNET_CADET_receive_done (ch);
-  }
-  else
-  {
-    schedule_reconnect (ch->cadet);
-  }
-}
-
-
-/**
- * Implementation function that cancels the currently sent message.
- * Should basically undo whatever #mq_send_impl() did.
- *
- * @param mq message queue
- * @param impl_state state specific to the implementation
- */
-static void
-cadet_mq_cancel_impl (struct GNUNET_MQ_Handle *mq,
-                     void *impl_state)
-{
-  struct GNUNET_CADET_Channel *ch = impl_state;
-
-  GNUNET_assert (NULL != ch->pending_env);
-  GNUNET_MQ_discard (ch->pending_env);
-  ch->pending_env = NULL;
-  if (NULL != ch->mq_cont)
-  {
-    GNUNET_SCHEDULER_cancel (ch->mq_cont);
-    ch->mq_cont = NULL;
-  }
-}
-
-
-/**
- * Process the new channel notification and add it to the channels in the handle
- *
- * @param h     The cadet handle
- * @param msg   A message with the details of the new incoming channel
- */
-static void
-handle_channel_created (void *cls,
-                        const struct GNUNET_CADET_LocalChannelCreateMessage *msg)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_Port *port;
-  const struct GNUNET_HashCode *port_number;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  ccn = msg->ccn;
-  port_number = &msg->port;
-  if (ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  port = find_port (h,
-                    port_number);
-  if (NULL == port)
-  {
-    /* We could have closed the port but the service didn't know about it yet
-     * This is not an error.
-     */
-    struct GNUNET_CADET_LocalChannelDestroyMessage *d_msg;
-    struct GNUNET_MQ_Envelope *env;
-
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "No handler for incoming channel %X (on port %s, recently closed?)\n",
-         ntohl (ccn.channel_of_client),
-         GNUNET_h2s (port_number));
-    env = GNUNET_MQ_msg (d_msg,
-                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
-    d_msg->ccn = msg->ccn;
-    GNUNET_MQ_send (h->mq,
-                    env);
-    return;
-  }
-
-  ch = create_channel (h,
-                       &ccn);
-  ch->peer = msg->peer;
-  ch->cadet = h;
-  ch->incoming_port = port;
-  ch->options = ntohl (msg->opt);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating incoming channel %X [%s] %p\n",
-       ntohl (ccn.channel_of_client),
-       GNUNET_h2s (port_number),
-       ch);
-
-  GNUNET_assert (NULL != port->connects);
-  ch->window_changes = port->window_changes;
-  ch->disconnects = port->disconnects;
-  ch->mq = GNUNET_MQ_queue_for_callbacks (&cadet_mq_send_impl,
-                                          &cadet_mq_destroy_impl,
-                                          &cadet_mq_cancel_impl,
-                                          ch,
-                                          port->handlers,
-                                          &cadet_mq_error_handler,
-                                          ch);
-  ch->ctx = port->connects (port->cls,
-                            ch,
-                            &msg->peer);
-  GNUNET_MQ_set_handlers_closure (ch->mq,
-                                  ch->ctx);
-}
-
-
-/**
- * Process the channel destroy notification and free associated resources
- *
- * @param h     The cadet handle
- * @param msg   A message with the details of the channel being destroyed
- */
-static void
-handle_channel_destroy (void *cls,
-                        const struct GNUNET_CADET_LocalChannelDestroyMessage *msg)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  struct GNUNET_CADET_Channel *ch;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received channel destroy for channel %X from CADET service\n",
-       ntohl (msg->ccn.channel_of_client));
-  ch = find_channel (h,
-                     msg->ccn);
-  if (NULL == ch)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  destroy_channel (ch);
-}
-
-
-/**
- * Check that message received from CADET service is well-formed.
- *
- * @param cls the `struct GNUNET_CADET_Handle`
- * @param message the message we got
- * @return #GNUNET_OK if the message is well-formed,
- *         #GNUNET_SYSERR otherwise
- */
-static int
-check_local_data (void *cls,
-                  const struct GNUNET_CADET_LocalData *message)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  struct GNUNET_CADET_Channel *ch;
-  uint16_t size;
-
-  size = ntohs (message->header.size);
-  if (sizeof (*message) + sizeof (struct GNUNET_MessageHeader) > size)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-
-  ch = find_channel (h,
-                     message->ccn);
-  if (NULL == ch)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Process the incoming data packets, call appropriate handlers.
- *
- * @param h       The cadet handle
- * @param message A message encapsulating the data
- */
-static void
-handle_local_data (void *cls,
-                   const struct GNUNET_CADET_LocalData *message)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  const struct GNUNET_MessageHeader *payload;
-  struct GNUNET_CADET_Channel *ch;
-  uint16_t type;
-  int fwd;
-
-  ch = find_channel (h,
-                     message->ccn);
-  if (NULL == ch)
-  {
-    GNUNET_break (0);
-    reconnect (h);
-    return;
-  }
-
-  payload = (const struct GNUNET_MessageHeader *) &message[1];
-  type = ntohs (payload->type);
-  fwd = ntohl (ch->ccn.channel_of_client) <= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got a %s data on channel %s [%X] of type %u\n",
-       fwd ? "FWD" : "BWD",
-       GNUNET_i2s (&ch->peer),
-       ntohl (message->ccn.channel_of_client),
-       type);
-  GNUNET_MQ_inject_message (ch->mq,
-                            payload);
-}
-
-
-/**
- * Process a local ACK message, enabling the client to send
- * more data to the service.
- *
- * @param h Cadet handle.
- * @param message Message itself.
- */
-static void
-handle_local_ack (void *cls,
-                  const struct GNUNET_CADET_LocalAck *message)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  struct GNUNET_CADET_Channel *ch;
-
-  ch = find_channel (h,
-                     message->ccn);
-  if (NULL == ch)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "ACK on unknown channel %X\n",
-         ntohl (message->ccn.channel_of_client));
-    return;
-  }
-  ch->allow_send++;
-  if (NULL == ch->pending_env)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got an ACK on mq channel %X, allow send now %u!\n",
-         ntohl (ch->ccn.channel_of_client),
-         ch->allow_send);
-    notify_window_size (ch);
-    return;
-  }
-  if (NULL != ch->mq_cont)
-    return; /* already working on it! */
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got an ACK on mq channel %X, sending pending message!\n",
-       ntohl (ch->ccn.channel_of_client));
-  ch->mq_cont
-    = GNUNET_SCHEDULER_add_now (&cadet_mq_send_now,
-                                ch);
-}
-
-
-/**
- * Generic error handler, called with the appropriate error code and
- * the same closure specified at the creation of the message queue.
- * Not every message queue implementation supports an error handler.
- *
- * @param cls closure, a `struct GNUNET_CORE_Handle *`
- * @param error error code
- */
-static void
-handle_mq_error (void *cls,
-                 enum GNUNET_MQ_Error error)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "MQ ERROR: %u\n",
-              error);
-  GNUNET_MQ_destroy (h->mq);
-  h->mq = NULL;
-  reconnect (h);
-}
-
-
-/**
- * Process a local reply about info on all tunnels, pass info to the user.
- *
- * @param cls Closure (Cadet handle).
- * @param msg Message itself.
- */
-static void
-handle_get_peers (void *cls,
-                  const struct GNUNET_CADET_LocalInfoPeer *msg)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-
-  if (NULL == h->info_cb.peers_cb)
-    return;
-  h->info_cb.peers_cb (h->info_cls,
-                       &msg->destination,
-                       (int) ntohs (msg->tunnel),
-                       (unsigned int) ntohs (msg->paths),
-                       0);
-}
-
-
-/**
- * Check that message received from CADET service is well-formed.
- *
- * @param cls the `struct GNUNET_CADET_Handle`
- * @param message the message we got
- * @return #GNUNET_OK if the message is well-formed,
- *         #GNUNET_SYSERR otherwise
- */
-static int
-check_get_peer (void *cls,
-                const struct GNUNET_CADET_LocalInfoPeer *message)
-{
-  size_t msize = sizeof (struct GNUNET_CADET_LocalInfoPeer);
-  const struct GNUNET_PeerIdentity *paths_array;
-  size_t esize;
-  unsigned int epaths;
-  unsigned int paths;
-  unsigned int peers;
-
-  esize = ntohs (message->header.size);
-  if (esize < msize)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  if (0 != ((esize - msize) % sizeof (struct GNUNET_PeerIdentity)))
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  peers = (esize - msize) / sizeof (struct GNUNET_PeerIdentity);
-  epaths = ntohs (message->paths);
-  paths_array = (const struct GNUNET_PeerIdentity *) &message[1];
-  paths = 0;
-  for (unsigned int i = 0; i < peers; i++)
-    if (0 == memcmp (&paths_array[i],
-                     &message->destination,
-                     sizeof (struct GNUNET_PeerIdentity)))
-      paths++;
-  if (paths != epaths)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Process a local peer info reply, pass info to the user.
- *
- * @param cls Closure (Cadet handle).
- * @param message Message itself.
- */
-static void
-handle_get_peer (void *cls,
-                 const struct GNUNET_CADET_LocalInfoPeer *message)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  const struct GNUNET_PeerIdentity *paths_array;
-  unsigned int paths;
-  unsigned int path_length;
-  int neighbor;
-  unsigned int peers;
-
-  if (NULL == h->info_cb.peer_cb)
-    return;
-  paths = ntohs (message->paths);
-  paths_array = (const struct GNUNET_PeerIdentity *) &message[1];
-  peers = (ntohs (message->header.size) - sizeof (*message))
-          / sizeof (struct GNUNET_PeerIdentity);
-  path_length = 0;
-  neighbor = GNUNET_NO;
-
-  for (unsigned int i = 0; i < peers; i++)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                " %s\n",
-                GNUNET_i2s (&paths_array[i]));
-    path_length++;
-    if (0 == memcmp (&paths_array[i], &message->destination,
-                     sizeof (struct GNUNET_PeerIdentity)))
-    {
-      if (1 == path_length)
-        neighbor = GNUNET_YES;
-      path_length = 0;
-    }
-  }
-
-  /* Call Callback with tunnel info. */
-  paths_array = (const struct GNUNET_PeerIdentity *) &message[1];
-  h->info_cb.peer_cb (h->info_cls,
-                      &message->destination,
-                      (int) ntohs (message->tunnel),
-                      neighbor,
-                      paths,
-                      paths_array);
-}
-
-
-/**
- * Process a local reply about info on all tunnels, pass info to the user.
- *
- * @param cls Closure (Cadet handle).
- * @param message Message itself.
- */
-static void
-handle_get_tunnels (void *cls,
-                    const struct GNUNET_CADET_LocalInfoTunnel *msg)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-
-  if (NULL == h->info_cb.tunnels_cb)
-    return;
-  h->info_cb.tunnels_cb (h->info_cls,
-                         &msg->destination,
-                         ntohl (msg->channels),
-                         ntohl (msg->connections),
-                         ntohs (msg->estate),
-                         ntohs (msg->cstate));
-
-}
-
-
-/**
- * Check that message received from CADET service is well-formed.
- *
- * @param cls the `struct GNUNET_CADET_Handle`
- * @param msg the message we got
- * @return #GNUNET_OK if the message is well-formed,
- *         #GNUNET_SYSERR otherwise
- */
-static int
-check_get_tunnel (void *cls,
-                  const struct GNUNET_CADET_LocalInfoTunnel *msg)
-{
-  unsigned int ch_n;
-  unsigned int c_n;
-  size_t esize;
-  size_t msize;
-
-  /* Verify message sanity */
-  msize = ntohs (msg->header.size);
-  esize = sizeof (struct GNUNET_CADET_LocalInfoTunnel);
-  if (esize > msize)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  ch_n = ntohl (msg->channels);
-  c_n = ntohl (msg->connections);
-  esize += ch_n * sizeof (struct GNUNET_CADET_ChannelTunnelNumber);
-  esize += c_n * sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier);
-  if (msize != esize)
-  {
-    GNUNET_break_op (0);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "m:%u, e: %u (%u ch, %u conn)\n",
-                (unsigned int) msize,
-                (unsigned int) esize,
-                ch_n,
-                c_n);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Process a local tunnel info reply, pass info to the user.
- *
- * @param cls Closure (Cadet handle).
- * @param msg Message itself.
- */
-static void
-handle_get_tunnel (void *cls,
-                   const struct GNUNET_CADET_LocalInfoTunnel *msg)
-{
-  struct GNUNET_CADET_Handle *h = cls;
-  unsigned int ch_n;
-  unsigned int c_n;
-  const struct GNUNET_CADET_ConnectionTunnelIdentifier *conns;
-  const struct GNUNET_CADET_ChannelTunnelNumber *chns;
-
-  if (NULL == h->info_cb.tunnel_cb)
-    return;
-
-  ch_n = ntohl (msg->channels);
-  c_n = ntohl (msg->connections);
-
-  /* Call Callback with tunnel info. */
-  conns = (const struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
-  chns = (const struct GNUNET_CADET_ChannelTunnelNumber *) &conns[c_n];
-  h->info_cb.tunnel_cb (h->info_cls,
-                        &msg->destination,
-                        ch_n,
-                        c_n,
-                        chns,
-                        conns,
-                        ntohs (msg->estate),
-                        ntohs (msg->cstate));
-}
-
-
-/**
- * Reconnect to the service, retransmit all infomation to try to restore the
- * original state.
- *
- * @param h handle to the cadet
- */
-static void
-reconnect (struct GNUNET_CADET_Handle *h)
-{
-  struct GNUNET_MQ_MessageHandler handlers[] = {
-    GNUNET_MQ_hd_fixed_size (channel_created,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE,
-                             struct GNUNET_CADET_LocalChannelCreateMessage,
-                             h),
-    GNUNET_MQ_hd_fixed_size (channel_destroy,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY,
-                             struct GNUNET_CADET_LocalChannelDestroyMessage,
-                             h),
-    GNUNET_MQ_hd_var_size (local_data,
-                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
-                           struct GNUNET_CADET_LocalData,
-                           h),
-    GNUNET_MQ_hd_fixed_size (local_ack,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
-                             struct GNUNET_CADET_LocalAck,
-                             h),
-    GNUNET_MQ_hd_fixed_size (get_peers,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
-                             struct GNUNET_CADET_LocalInfoPeer,
-                             h),
-    GNUNET_MQ_hd_var_size (get_peer,
-                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
-                           struct GNUNET_CADET_LocalInfoPeer,
-                           h),
-    GNUNET_MQ_hd_fixed_size (get_tunnels,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
-                             struct GNUNET_CADET_LocalInfoTunnel,
-                             h),
-    GNUNET_MQ_hd_var_size (get_tunnel,
-                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
-                           struct GNUNET_CADET_LocalInfoTunnel,
-                           h),
-    GNUNET_MQ_handler_end ()
-  };
-
-  h->mq = GNUNET_CLIENT_connect (h->cfg,
-                                 "cadet",
-                                 handlers,
-                                 &handle_mq_error,
-                                 h);
-  if (NULL == h->mq)
-  {
-    schedule_reconnect (h);
-    return;
-  }
-  h->reconnect_time = GNUNET_TIME_UNIT_MILLISECONDS;
-}
-
-
-/**
- * Function called during #GNUNET_CADET_disconnect() to destroy
- * all channels that are still open.
- *
- * @param cls the `struct GNUNET_CADET_Handle`
- * @param cid chanenl ID
- * @param value a `struct GNUNET_CADET_Channel` to destroy
- * @return #GNUNET_OK (continue to iterate)
- */
-static int
-destroy_channel_cb (void *cls,
-                    uint32_t cid,
-                    void *value)
-{
-  /* struct GNUNET_CADET_Handle *handle = cls; */
-  struct GNUNET_CADET_Channel *ch = value;
-
-  if (ntohl (ch->ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-  {
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "channel %X not destroyed\n",
-         ntohl (ch->ccn.channel_of_client));
-  }
-  destroy_channel (ch);
-  return GNUNET_OK;
-}
-
-
-/**
- * Function called during #GNUNET_CADET_disconnect() to destroy
- * all ports that are still open.
- *
- * @param cls the `struct GNUNET_CADET_Handle`
- * @param id port ID
- * @param value a `struct GNUNET_CADET_Channel` to destroy
- * @return #GNUNET_OK (continue to iterate)
- */
-static int
-destroy_port_cb (void *cls,
-                 const struct GNUNET_HashCode *id,
-                 void *value)
-{
-  /* struct GNUNET_CADET_Handle *handle = cls; */
-  struct GNUNET_CADET_Port *port = value;
-
-  /* This is a warning, the app should have cleanly closed all open ports */
-  GNUNET_break (0);
-  GNUNET_CADET_close_port (port);
-  return GNUNET_OK;
-}
-
-
-/**
- * Disconnect from the cadet service. All channels will be destroyed. All channel
- * disconnect callbacks will be called on any still connected peers, notifying
- * about their disconnection. The registered inbound channel cleaner will be
- * called should any inbound channels still exist.
- *
- * @param handle connection to cadet to disconnect
- */
-void
-GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle)
-{
-  GNUNET_CONTAINER_multihashmap_iterate (handle->ports,
-                                         &destroy_port_cb,
-                                         handle);
-  GNUNET_CONTAINER_multihashmap_destroy (handle->ports);
-  handle->ports = NULL;
-  GNUNET_CONTAINER_multihashmap32_iterate (handle->channels,
-                                           &destroy_channel_cb,
-                                           handle);
-  GNUNET_CONTAINER_multihashmap32_destroy (handle->channels);
-  handle->channels = NULL;
-  if (NULL != handle->mq)
-  {
-    GNUNET_MQ_destroy (handle->mq);
-    handle->mq = NULL;
-  }
-  if (NULL != handle->reconnect_task)
-  {
-    GNUNET_SCHEDULER_cancel (handle->reconnect_task);
-    handle->reconnect_task = NULL;
-  }
-  GNUNET_free (handle);
-}
-
-
-/**
- * Close a port opened with @a GNUNET_CADET_open_port().
- * The @a new_channel callback will no longer be called.
- *
- * @param p Port handle.
- */
-void
-GNUNET_CADET_close_port (struct GNUNET_CADET_Port *p)
-{
-  struct GNUNET_CADET_PortMessage *msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE);
-  msg->port = p->id;
-  GNUNET_MQ_send (p->cadet->mq,
-                  env);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (p->cadet->ports,
-                                                       &p->id,
-                                                       p));
-  GNUNET_free_non_null (p->handlers);
-  GNUNET_free (p);
-}
-
-
-/**
- * Destroy an existing channel.
- *
- * The existing end callback for the channel will be called immediately.
- * Any pending outgoing messages will be sent but no incoming messages will be
- * accepted and no data callbacks will be called.
- *
- * @param channel Channel handle, becomes invalid after this call.
- */
-void
-GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel)
-{
-  struct GNUNET_CADET_Handle *h = channel->cadet;
-  struct GNUNET_CADET_LocalChannelDestroyMessage *msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  if (NULL != h->mq)
-  {
-    env = GNUNET_MQ_msg (msg,
-                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
-    msg->ccn = channel->ccn;
-    GNUNET_MQ_send (h->mq,
-                    env);
-  }
-  destroy_channel (channel);
-}
-
-
-/**
- * Get information about a channel.
- *
- * @param channel Channel handle.
- * @param option Query (GNUNET_CADET_OPTION_*).
- * @param ... dependant on option, currently not used
- *
- * @return Union with an answer to the query.
- */
-const union GNUNET_CADET_ChannelInfo *
-GNUNET_CADET_channel_get_info (struct GNUNET_CADET_Channel *channel,
-                               enum GNUNET_CADET_ChannelOption option,
-                               ...)
-{
-  static int bool_flag;
-
-  switch (option)
-  {
-    case GNUNET_CADET_OPTION_NOBUFFER:
-    case GNUNET_CADET_OPTION_RELIABLE:
-    case GNUNET_CADET_OPTION_OUT_OF_ORDER:
-      if (0 != (option & channel->options))
-        bool_flag = GNUNET_YES;
-      else
-        bool_flag = GNUNET_NO;
-      return (const union GNUNET_CADET_ChannelInfo *) &bool_flag;
-      break;
-    case GNUNET_CADET_OPTION_PEER:
-      return (const union GNUNET_CADET_ChannelInfo *) &channel->peer;
-      break;
-    default:
-      GNUNET_break (0);
-      return NULL;
-  }
-}
-
-
-/**
- * Send an ack on the channel to confirm the processing of a message.
- *
- * @param ch Channel on which to send the ACK.
- */
-void
-GNUNET_CADET_receive_done (struct GNUNET_CADET_Channel *channel)
-{
-  struct GNUNET_CADET_LocalAck *msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending ACK on channel %X\n",
-       ntohl (channel->ccn.channel_of_client));
-  msg->ccn = channel->ccn;
-  GNUNET_MQ_send (channel->cadet->mq,
-                  env);
-}
-
-
-/**
- * Send message of @a type to CADET service of @a h
- *
- * @param h handle to CADET service
- * @param type message type of trivial information request to send
- */
-static void
-send_info_request (struct GNUNET_CADET_Handle *h,
-                   uint16_t type)
-{
-  struct GNUNET_MessageHeader *msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  env = GNUNET_MQ_msg (msg,
-                       type);
-  GNUNET_MQ_send (h->mq,
-                  env);
-}
-
-
-/**
- * Request a debug dump on the service's STDERR.
- *
- * WARNING: unstable API, likely to change in the future!
- *
- * @param h cadet handle
- */
-void
-GNUNET_CADET_request_dump (struct GNUNET_CADET_Handle *h)
-{
-  send_info_request (h,
-                     GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP);
-}
-
-
-/**
- * Request information about peers known to the running cadet service.
- * The callback will be called for every peer known to the service.
- * Only one info request (of any kind) can be active at once.
- *
- * WARNING: unstable API, likely to change in the future!
- *
- * @param h Handle to the cadet peer.
- * @param callback Function to call with the requested data.
- * @param callback_cls Closure for @c callback.
- * @return #GNUNET_OK / #GNUNET_SYSERR
- */
-int
-GNUNET_CADET_get_peers (struct GNUNET_CADET_Handle *h,
-                       GNUNET_CADET_PeersCB callback,
-                       void *callback_cls)
-{
-  if (NULL != h->info_cb.peers_cb)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  send_info_request (h,
-                     GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
-  h->info_cb.peers_cb = callback;
-  h->info_cls = callback_cls;
-  return GNUNET_OK;
-}
-
-
-/**
- * Cancel a peer info request. The callback will not be called (anymore).
- *
- * WARNING: unstable API, likely to change in the future!
- *
- * @param h Cadet handle.
- * @return Closure given to GNUNET_CADET_get_peers().
- */
-void *
-GNUNET_CADET_get_peers_cancel (struct GNUNET_CADET_Handle *h)
-{
-  void *cls = h->info_cls;
-
-  h->info_cb.peers_cb = NULL;
-  h->info_cls = NULL;
-  return cls;
-}
-
-
-/**
- * Request information about a peer known to the running cadet peer.
- * The callback will be called for the tunnel once.
- * Only one info request (of any kind) can be active at once.
- *
- * WARNING: unstable API, likely to change in the future!
- *
- * @param h Handle to the cadet peer.
- * @param id Peer whose tunnel to examine.
- * @param callback Function to call with the requested data.
- * @param callback_cls Closure for @c callback.
- * @return #GNUNET_OK / #GNUNET_SYSERR
- */
-int
-GNUNET_CADET_get_peer (struct GNUNET_CADET_Handle *h,
-                       const struct GNUNET_PeerIdentity *id,
-                       GNUNET_CADET_PeerCB callback,
-                       void *callback_cls)
-{
-  struct GNUNET_CADET_LocalInfo *msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  if (NULL != h->info_cb.peer_cb)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
-  msg->peer = *id;
-  GNUNET_MQ_send (h->mq,
-                  env);
-  h->info_cb.peer_cb = callback;
-  h->info_cls = callback_cls;
-  return GNUNET_OK;
-}
-
-
-/**
- * Request information about tunnels of the running cadet peer.
- * The callback will be called for every tunnel of the service.
- * Only one info request (of any kind) can be active at once.
- *
- * WARNING: unstable API, likely to change in the future!
- *
- * @param h Handle to the cadet peer.
- * @param callback Function to call with the requested data.
- * @param callback_cls Closure for @c callback.
- * @return #GNUNET_OK / #GNUNET_SYSERR
- */
-int
-GNUNET_CADET_get_tunnels (struct GNUNET_CADET_Handle *h,
-                         GNUNET_CADET_TunnelsCB callback,
-                         void *callback_cls)
-{
-  if (NULL != h->info_cb.tunnels_cb)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  send_info_request (h,
-                     GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
-  h->info_cb.tunnels_cb = callback;
-  h->info_cls = callback_cls;
-  return GNUNET_OK;
-}
-
-
-/**
- * Cancel a monitor request. The monitor callback will not be called.
- *
- * @param h Cadet handle.
- * @return Closure given to GNUNET_CADET_get_tunnels().
- */
-void *
-GNUNET_CADET_get_tunnels_cancel (struct GNUNET_CADET_Handle *h)
-{
-  void *cls = h->info_cls;
-
-  h->info_cb.tunnels_cb = NULL;
-  h->info_cls = NULL;
-  return cls;
-}
-
-
-/**
- * Request information about a tunnel of the running cadet peer.
- * The callback will be called for the tunnel once.
- * Only one info request (of any kind) can be active at once.
- *
- * WARNING: unstable API, likely to change in the future!
- *
- * @param h Handle to the cadet peer.
- * @param id Peer whose tunnel to examine.
- * @param callback Function to call with the requested data.
- * @param callback_cls Closure for @c callback.
- * @return #GNUNET_OK / #GNUNET_SYSERR
- */
-int
-GNUNET_CADET_get_tunnel (struct GNUNET_CADET_Handle *h,
-                        const struct GNUNET_PeerIdentity *id,
-                        GNUNET_CADET_TunnelCB callback,
-                        void *callback_cls)
-{
-  struct GNUNET_CADET_LocalInfo *msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  if (NULL != h->info_cb.tunnel_cb)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
-  msg->peer = *id;
-  GNUNET_MQ_send (h->mq,
-                  env);
-  h->info_cb.tunnel_cb = callback;
-  h->info_cls = callback_cls;
-  return GNUNET_OK;
-}
-
-
-/**
- * Transitional function to convert an unsigned int port to a hash value.
- * WARNING: local static value returned, NOT reentrant!
- * WARNING: do not use this function for new code!
- *
- * @param port Numerical port (unsigned int format).
- *
- * @return A GNUNET_HashCode usable for the new CADET API.
- */
-const struct GNUNET_HashCode *
-GC_u2h (uint32_t port)
-{
-  static struct GNUNET_HashCode hash;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-              "This is a transitional function, use proper crypto hashes as CADET ports\n");
-  GNUNET_CRYPTO_hash (&port,
-                      sizeof (port),
-                      &hash);
-  return &hash;
-}
-
-
-/**
- * Connect to the MQ-based cadet service.
- *
- * @param cfg Configuration to use.
- *
- * @return Handle to the cadet service NULL on error.
- */
-struct GNUNET_CADET_Handle *
-GNUNET_CADET_connecT (const struct GNUNET_CONFIGURATION_Handle *cfg)
-{
-  struct GNUNET_CADET_Handle *h;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "GNUNET_CADET_connecT()\n");
-  h = GNUNET_new (struct GNUNET_CADET_Handle);
-  h->cfg = cfg;
-  h->ports = GNUNET_CONTAINER_multihashmap_create (4,
-                                                   GNUNET_YES);
-  h->channels = GNUNET_CONTAINER_multihashmap32_create (4);
-  reconnect (h);
-  if (NULL == h->mq)
-  {
-    GNUNET_break (0);
-    GNUNET_CADET_disconnect (h);
-    return NULL;
-  }
-  h->next_ccn.channel_of_client = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-  h->reconnect_time = GNUNET_TIME_UNIT_MILLISECONDS;
-  h->reconnect_task = NULL;
-
-  return h;
-}
-
-
-/**
- * Open a port to receive incomming MQ-based channels.
- *
- * @param h CADET handle.
- * @param port Hash identifying the port.
- * @param connects Function called when an incoming channel is connected.
- * @param connects_cls Closure for the @a connects handler.
- * @param window_changes Function called when the transmit window size changes.
- * @param disconnects Function called when a channel is disconnected.
- * @param handlers Callbacks for messages we care about, NULL-terminated.
- * @return Port handle.
- */
-struct GNUNET_CADET_Port *
-GNUNET_CADET_open_porT (struct GNUNET_CADET_Handle *h,
-                        const struct GNUNET_HashCode *port,
-                        GNUNET_CADET_ConnectEventHandler connects,
-                        void * connects_cls,
-                        GNUNET_CADET_WindowSizeEventHandler window_changes,
-                        GNUNET_CADET_DisconnectEventHandler disconnects,
-                        const struct GNUNET_MQ_MessageHandler *handlers)
-{
-  struct GNUNET_CADET_PortMessage *msg;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_Port *p;
-
-  GNUNET_assert (NULL != connects);
-  GNUNET_assert (NULL != disconnects);
-
-  p = GNUNET_new (struct GNUNET_CADET_Port);
-  p->cadet = h;
-  p->id = *port;
-  p->connects = connects;
-  p->cls = connects_cls;
-  p->window_changes = window_changes;
-  p->disconnects = disconnects;
-  p->handlers = GNUNET_MQ_copy_handlers (handlers);
-
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multihashmap_put (h->ports,
-                                                    &p->id,
-                                                    p,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN);
-  msg->port = p->id;
-  GNUNET_MQ_send (h->mq,
-                  env);
-  return p;
-}
-
-
-/**
- * Create a new channel towards a remote peer.
- *
- * If the destination port is not open by any peer or the destination peer
- * does not accept the channel, #GNUNET_CADET_ChannelEndHandler will be called
- * for this channel.
- *
- * @param h CADET handle.
- * @param channel_cls Closure for the channel. It's given to:
- *                    - The disconnect handler @a disconnects
- *                    - Each message type callback in @a handlers
- * @param destination Peer identity the channel should go to.
- * @param port Identification of the destination port.
- * @param options CadetOption flag field, with all desired option bits set to 1.
- * @param window_changes Function called when the transmit window size changes.
- * @param disconnects Function called when the channel is disconnected.
- * @param handlers Callbacks for messages we care about, NULL-terminated.
- * @return Handle to the channel.
- */
-struct GNUNET_CADET_Channel *
-GNUNET_CADET_channel_creatE (struct GNUNET_CADET_Handle *h,
-                             void *channel_cls,
-                             const struct GNUNET_PeerIdentity *destination,
-                             const struct GNUNET_HashCode *port,
-                             enum GNUNET_CADET_ChannelOption options,
-                             GNUNET_CADET_WindowSizeEventHandler window_changes,
-                             GNUNET_CADET_DisconnectEventHandler disconnects,
-                             const struct GNUNET_MQ_MessageHandler *handlers)
-{
-  struct GNUNET_CADET_Channel *ch;
-  struct GNUNET_CADET_LocalChannelCreateMessage *msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  GNUNET_assert (NULL != disconnects);
-  ch = create_channel (h,
-                       NULL);
-  ch->ctx = channel_cls;
-  ch->peer = *destination;
-  ch->options = options;
-  ch->window_changes = window_changes;
-  ch->disconnects = disconnects;
-
-  /* Create MQ for channel */
-  ch->mq = GNUNET_MQ_queue_for_callbacks (&cadet_mq_send_impl,
-                                          &cadet_mq_destroy_impl,
-                                          &cadet_mq_cancel_impl,
-                                          ch,
-                                          handlers,
-                                          &cadet_mq_error_handler,
-                                          ch);
-  GNUNET_MQ_set_handlers_closure (ch->mq, channel_cls);
-
-  /* Request channel creation to service */
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
-  msg->ccn = ch->ccn;
-  msg->port = *port;
-  msg->peer = *destination;
-  msg->opt = htonl (options);
-  GNUNET_MQ_send (h->mq,
-                  env);
-  return ch;
-}
-
-
-/**
- * Obtain the message queue for a connected peer.
- *
- * @param channel The channel handle from which to get the MQ.
- *
- * @return NULL if @a channel is not yet connected.
- */
-struct GNUNET_MQ_Handle *
-GNUNET_CADET_get_mq (const struct GNUNET_CADET_Channel *channel)
-{
-  return channel->mq;
-}
-
-/* end of cadet_api.c */
diff --git a/src/cadet/cadet_common.c b/src/cadet/cadet_common.c
deleted file mode 100644
index 95a3144e4..000000000
--- a/src/cadet/cadet_common.c
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2012 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/cadet_common.c
- * @brief CADET helper functions
- * @author Bartlomiej Polot
- */
-
-#include "cadet.h"
-
-/**
- * @brief Translate a fwd variable into a string representation, for logging.
- *
- * @param fwd Is FWD? (#GNUNET_YES or #GNUNET_NO)
- *
- * @return String representing FWD or BCK.
- */
-char *
-GC_f2s (int fwd)
-{
-  if (GNUNET_YES == fwd)
-  {
-    return "FWD";
-  }
-  else if (GNUNET_NO == fwd)
-  {
-    return "BCK";
-  }
-  else
-  {
-    /* Not an error, can happen with CONNECTION_BROKEN messages. */
-    return "\???";
-  }
-}
-
-
-/**
- * Test if @a bigger is larger than @a smaller.
- * Considers the case that @a bigger just overflowed
- * and is thus tiny while @a smaller is still below
- * `UINT32_MAX`.
- */
-int
-GC_is_pid_bigger (uint32_t bigger,
-                  uint32_t smaller)
-{
-    return (PID_OVERFLOW (smaller, bigger) ||
-            ( (bigger > smaller) &&
-              (! PID_OVERFLOW (bigger, smaller))) );
-}
-
-
-uint32_t
-GC_max_pid (uint32_t a, uint32_t b)
-{
-  if (GC_is_pid_bigger(a, b))
-    return a;
-  return b;
-}
-
-
-uint32_t
-GC_min_pid (uint32_t a, uint32_t b)
-{
-  if (GC_is_pid_bigger(a, b))
-    return b;
-  return a;
-}
-
-
-/**
- * Allocate a string with a hexdump of any binary data.
- *
- * @param bin Arbitrary binary data.
- * @param len Length of @a bin in bytes.
- * @param output Where to write the output (if *output be NULL it's allocated).
- *
- * @return The size of the output.
- */
-size_t
-GC_bin2s (void *bin, unsigned int len, char **output)
-{
-  char *data = bin;
-  char *buf;
-  unsigned int s_len;
-  unsigned int i;
-
-  s_len = 2 * len + 1;
-  if (NULL == *output)
-    *output = GNUNET_malloc (s_len);
-  buf = *output;
-
-  for (i = 0; i < len; i++)
-  {
-    SPRINTF (&buf[2 * i], "%2X", data[i]);
-  }
-  buf[s_len - 1] = '\0';
-
-  return s_len;
-}
-
-
-#if !defined(GNUNET_CULL_LOGGING)
-const char *
-GC_m2s (uint16_t m)
-{
-  static char buf[2][16];
-  static int idx;
-  const char *s;
-
-  idx = (idx + 1) % 2;
-  switch (m)
-  {
-    /**
-     * Used to mark the "payload" of a non-payload message.
-     */
-    case 0:
-      s = "retransmit";
-      break;
-
-      /**
-       * Request the creation of a path
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE:
-      s = "CONN_CREAT";
-      break;
-
-      /**
-       * Request the modification of an existing path
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK:
-      s = "CONN_ACK";
-      break;
-
-      /**
-       * Notify that a connection of a path is no longer valid
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN:
-      s = "CONN_BRKN";
-      break;
-
-      /**
-       * At some point, the route will spontaneously change
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_PATH_CHANGED_UNIMPLEMENTED:
-      s = "PATH_CHNGD";
-      break;
-
-      /**
-       * Transport payload data.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-      s = "DATA";
-      break;
-
-    /**
-     * Confirm receipt of payload data.
-     */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-      s = "DATA_ACK";
-      break;
-
-      /**
-       * Key exchange message.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX:
-      s = "KX";
-      break;
-
-      /**
-       * Encrypted.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED:
-      s = "ENCRYPTED";
-      break;
-
-      /**
-       * Request the destuction of a path
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY:
-      s = "CONN_DSTRY";
-      break;
-
-      /**
-       * ACK for a data packet.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK:
-      s = "ACK";
-      break;
-
-      /**
-       * POLL for ACK.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL:
-      s = "POLL";
-      break;
-
-      /**
-       * Announce origin is still alive.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE:
-      s = "KEEPALIVE";
-      break;
-
-      /**
-       * Open port
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN:
-      s = "OPEN_PORT";
-      break;
-
-      /**
-       * Close port
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE:
-      s = "CLOSE_PORT";
-      break;
-
-      /**
-       * Ask the cadet service to create a new tunnel
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-      s = "CHAN_CREAT";
-      break;
-
-      /**
-       * Ask the cadet service to destroy a tunnel
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-      s = "CHAN_DSTRY";
-      break;
-
-      /**
-       * Confirm the creation of a channel.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-      s = "CHAN_ACK";
-      break;
-
-      /**
-       * Confirm the creation of a channel.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED:
-      s = "CHAN_NACK";
-      break;
-
-      /**
-       * Local payload traffic
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA:
-      s = "LOC_DATA";
-      break;
-
-      /**
-       * Local ACK for data.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK:
-      s = "LOC_ACK";
-      break;
-
-      /**
-       * Local monitoring of channels.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNELS:
-      s = "INFO_CHANS";
-      break;
-
-      /**
-       * Local monitoring of a channel.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNEL:
-      s = "INFO_CHAN";
-      break;
-
-      /**
-       * Local monitoring of service.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS:
-      s = "INFO_TUNS";
-      break;
-
-      /**
-       * Local monitoring of service.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL:
-      s = "INFO_TUN";
-      break;
-
-      /**
-       * Local information about all connections of service.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CONNECTIONS:
-      s = "INFO_CONNS";
-      break;
-
-      /**
-       * Local information of service about a specific connection.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CONNECTION:
-      s = "INFO_CONN";
-      break;
-
-      /**
-       * Local information about all peers known to the service.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS:
-      s = "INFO_PEERS";
-      break;
-
-      /**
-       * Local information of service about a specific peer.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER:
-      s = "INFO_PEER";
-      break;
-
-      /**
-       * Traffic (net-cat style) used by the Command Line Interface.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_CLI:
-      s = "CLI";
-      break;
-
-      /**
-       * Debug request.
-       */
-    case GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP:
-      s = "INFO_DUMP";
-      break;
-
-      /**
-       * Used to mark the "payload" of a non-payload message.
-       */
-    case UINT16_MAX:
-      s = "      N/A";
-      break;
-
-
-    default:
-      SPRINTF (buf[idx], "{UNK: %5u}", m);
-      return buf[idx];
-  }
-  SPRINTF (buf[idx], "{%10s}", s);
-  return buf[idx];
-}
-#else
-const char *
-GC_m2s (uint16_t m)
-{
-  return "";
-}
-#endif
diff --git a/src/cadet/cadet_path.c b/src/cadet/cadet_path.c
deleted file mode 100644
index 79a498805..000000000
--- a/src/cadet/cadet_path.c
+++ /dev/null
@@ -1,363 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001 - 2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/cadet_path.c
- * @brief Path handling functions
- * @author Bartlomiej Polot
- */
-
-#include "cadet.h"
-#include "cadet_path.h"
-#include "gnunet-service-cadet_peer.h"
-
-#define LOG(level, ...) GNUNET_log_from (level,"cadet-pth",__VA_ARGS__)
-
-
-/**
- * @brief Destroy a path after some time has past.
- * Removes the path from the peer (must not be used for direct paths).
- *
- * @param cls Closure (path to destroy).
- */
-static void
-path_destroy_delayed (void *cls)
-{
-  struct CadetPeerPath *path = cls;
-  struct CadetPeer *peer;
-
-  path->path_delete = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroy delayed %p (%u)\n",
-       path,
-       path->length);
-  GNUNET_assert (2 < path->length);
-  peer = GCP_get_short (path->peers[path->length - 1],
-                        GNUNET_NO);
-  GNUNET_assert (NULL != peer);
-  GCP_remove_path (peer, path);
-}
-
-
-/**
- * Create a new path
- *
- * @param length How many hops will the path have.
- * @return A newly allocated path with a peer array of the specified length.
- */
-struct CadetPeerPath *
-path_new (unsigned int length)
-{
-  struct CadetPeerPath *p;
-
-  p = GNUNET_new (struct CadetPeerPath);
-  if (length > 0)
-  {
-    p->length = length;
-    p->peers = GNUNET_malloc (length * sizeof (GNUNET_PEER_Id));
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "New path %p (%u)\n", p, p->length);
-  return p;
-}
-
-
-/**
- * Invert the path
- *
- * @param path the path to invert
- */
-void
-path_invert (struct CadetPeerPath *path)
-{
-  GNUNET_PEER_Id aux;
-  unsigned int i;
-
-  for (i = 0; i < path->length / 2; i++)
-  {
-    aux = path->peers[i];
-    path->peers[i] = path->peers[path->length - i - 1];
-    path->peers[path->length - i - 1] = aux;
-  }
-}
-
-
-/**
- * Duplicate a path, incrementing short peer's rc.
- *
- * @param path The path to duplicate.
- */
-struct CadetPeerPath *
-path_duplicate (const struct CadetPeerPath *path)
-{
-  struct CadetPeerPath *aux;
-  unsigned int i;
-
-  aux = path_new (path->length);
-  GNUNET_memcpy (aux->peers,
-          path->peers,
-          path->length * sizeof (GNUNET_PEER_Id));
-  for (i = 0; i < aux->length; i++)
-    GNUNET_PEER_change_rc (aux->peers[i], 1);
-  return aux;
-}
-
-
-/**
- * Get the length of a path.
- *
- * @param path The path to measure, with the local peer at any point of it.
- *
- * @return Number of hops to reach destination.
- *         UINT_MAX in case the peer is not in the path.
- */
-unsigned int
-path_get_length (struct CadetPeerPath *path)
-{
-  if (NULL == path)
-    return UINT_MAX;
-  return path->length;
-}
-
-
-
-/**
- * Mark path as invalid: keep it aroud for a while to avoid trying it in a loop.
- *
- * Never invalidates a two-hop (direct) path, only a core handler can do that.
- *
- * Rationale: DHT_get sometimes returns bad cached results, for instance,
- * on a locally cached result where the PUT followed a path that is no longer
- * current. The path must remain "known and marked as invalid" for a while.
- *
- * @param p Path to invalidate.
- */
-void
-path_invalidate (struct CadetPeerPath *p)
-{
-  if (NULL != p->path_delete)
-    return;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Invalidating path %p (%u)\n",
-       p,
-       p->length);
-  p->path_delete
-    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                    &path_destroy_delayed, p);
-}
-
-
-/**
- * Builds a path from a PeerIdentity array.
- *
- * @param peers PeerIdentity array.
- * @param size Size of the @c peers array.
- * @param myid ID of local peer, to find @c own_pos.
- * @param own_pos Output parameter: own position in the path.
- *
- * @return Fixed and shortened path.
- */
-struct CadetPeerPath *
-path_build_from_peer_ids (struct GNUNET_PeerIdentity *peers,
-                          unsigned int size,
-                          GNUNET_PEER_Id myid,
-                          unsigned int *own_pos)
-{
-  struct CadetPeerPath *path;
-  GNUNET_PEER_Id shortid;
-  unsigned int i;
-  unsigned int j;
-  unsigned int offset;
-
-  /* Create path */
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  Creating path...\n");
-  path = path_new (size);
-  *own_pos = 0;
-  offset = 0;
-  for (i = 0; i < size; i++)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  - %u: taking %s\n",
-         i, GNUNET_i2s (&peers[i]));
-    shortid = GNUNET_PEER_intern (&peers[i]);
-
-    /* Check for loops / duplicates */
-    for (j = 0; j < i - offset; j++)
-    {
-      if (path->peers[j] == shortid)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "    already exists at pos %u\n", j);
-        offset = i - j;
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "    offset now %u\n", offset);
-        GNUNET_PEER_change_rc (shortid, -1);
-      }
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "    storing at %u\n", i - offset);
-    path->peers[i - offset] = shortid;
-    if (path->peers[i - offset] == myid)
-      *own_pos = i - offset;
-  }
-  path->length -= offset;
-
-  if (path->peers[*own_pos] != myid)
-  {
-    /* create path: self not found in path through self */
-    GNUNET_break_op (0);
-    path_destroy (path);
-    return NULL;
-  }
-
-  return path;
-}
-
-
-/**
- * Test if two paths are equivalent (equal or revese of each other).
- *
- * @param p1 First path
- * @param p2 Second path
- *
- * @return #GNUNET_YES if both paths are equivalent
- *         #GNUNET_NO otherwise
- */
-int
-path_equivalent (const struct CadetPeerPath *p1,
-                 const struct CadetPeerPath *p2)
-{
-  unsigned int i;
-  unsigned int l;
-  unsigned int half;
-
-  if (NULL == p1 || NULL == p2)
-    return GNUNET_NO;
-
-  if (p1->length != p2->length)
-    return GNUNET_NO;
-
-  l = p1->length;
-  if (0 == memcmp (p1->peers, p2->peers, sizeof (p1->peers[0]) * l))
-    return GNUNET_YES;
-
-  half = l / 2;
-  l = l - 1;
-  for (i = 0; i <= half; i++)
-    if (p1->peers[i] != p2->peers[l - i])
-      return GNUNET_NO;
-
-  return GNUNET_YES;
-}
-
-
-/**
- * Test if a path is valid (or at least not known to be invalid).
- *
- * @param path Path to test.
- *
- * @return #GNUNET_YES If the path is valid or unknown,
- *         #GNUNET_NO If the path is known to be invalid.
- */
-int
-path_is_valid (const struct CadetPeerPath *path)
-{
-  return (NULL == path->path_delete);
-}
-
-
-/**
- * Destroy the path and free any allocated resources linked to it
- *
- * @param p the path to destroy
- *
- * @return #GNUNET_OK on success
- */
-int
-path_destroy (struct CadetPeerPath *p)
-{
-  if (NULL == p)
-    return GNUNET_OK;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "destroying path %p (%u)\n",
-       p,
-       p->length);
-  GNUNET_PEER_decrement_rcs (p->peers, p->length);
-  GNUNET_free_non_null (p->peers);
-  if (NULL != p->path_delete)
-    GNUNET_SCHEDULER_cancel (p->path_delete);
-  GNUNET_free (p);
-  return GNUNET_OK;
-}
-
-
-/**
- * Compare two paths.
- *
- * @param p1 First path.
- * @param p2 Second path.
- *
- * @return > 0 if p1 is longer, or the first differing PEER_Id is higher on p1.
- *         < 0 if p2 is longer, or the first differing PEER_Id is higher on p2.
- *         0 if they are identical.
- */
-int
-path_cmp (const struct CadetPeerPath *p1,
-          const struct CadetPeerPath *p2)
-{
-  if (p1->length > p2->length)
-    return 1;
-
-  if (p1->length < p2->length)
-    return -1;
-
-  return memcmp (p1->peers,
-                 p2->peers,
-                 sizeof (GNUNET_PEER_Id) * p1->length);
-}
-
-
-char *
-path_2s (struct CadetPeerPath *p)
-{
-  char *s;
-  char *old;
-  unsigned int i;
-
-  old = GNUNET_strdup ("");
-  for (i = 0; i < p->length; i++)
-  {
-    GNUNET_asprintf (&s, "%s %s",
-                     old, GNUNET_i2s (GNUNET_PEER_resolve2 (p->peers[i])));
-    GNUNET_free_non_null (old);
-    old = s;
-  }
-  return old;
-}
-
-
-void
-path_debug (struct CadetPeerPath *p)
-{
-  unsigned int i;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "PATH:\n");
-  for (i = 0; i < p->length; i++)
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  %s\n",
-                GNUNET_i2s (GNUNET_PEER_resolve2 (p->peers[i])));
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "END\n");
-}
diff --git a/src/cadet/cadet_path.h b/src/cadet/cadet_path.h
deleted file mode 100644
index bb68eec42..000000000
--- a/src/cadet/cadet_path.h
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001 - 2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/cadet_path.h
- * @brief Path handling functions
- * @author Bartlomiej Polot
- */
-
-#ifndef CADET_PATH_H_
-#define CADET_PATH_H_
-
-#ifdef __cplusplus
-extern "C"
-{
-  #if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-
-/******************************************************************************/
-/************************      DATA STRUCTURES     ****************************/
-/******************************************************************************/
-
-/**
- * Information regarding a possible path to reach a single peer
- */
-struct CadetPeerPath
-{
-
-    /**
-     * Linked list
-     */
-  struct CadetPeerPath *next;
-  struct CadetPeerPath *prev;
-
-    /**
-     * List of all the peers that form the path from origin to target.
-     */
-  GNUNET_PEER_Id *peers;
-
-    /**
-     * Number of peers (hops) in the path
-     */
-  unsigned int length;
-
-    /**
-     * User defined data store.
-     */
-  struct CadetConnection *c;
-
-    /**
-     * Path's score, how reliable is the path.
-     */
-//   int score;
-
-  /**
-   * Task to delete the path.
-   * We tried it, it didn't work, don't try again in a while.
-   */
-  struct GNUNET_SCHEDULER_Task * path_delete;
-
-};
-
-/******************************************************************************/
-/*************************        FUNCTIONS       *****************************/
-/******************************************************************************/
-
-/**
- * Create a new path.
- *
- * @param length How many hops will the path have.
- *
- * @return A newly allocated path with a peer array of the specified length.
- */
-struct CadetPeerPath *
-path_new (unsigned int length);
-
-
-/**
- * Invert the path.
- *
- * @param path The path to invert.
- */
-void
-path_invert (struct CadetPeerPath *path);
-
-
-/**
- * Duplicate a path, incrementing short peer's rc.
- *
- * @param path The path to duplicate.
- */
-struct CadetPeerPath *
-path_duplicate (const struct CadetPeerPath *path);
-
-
-/**
- * Get the length of a path.
- *
- * @param path The path to measure, with the local peer at any point of it.
- *
- * @return Number of hops to reach destination.
- *         UINT_MAX in case the peer is not in the path.
- */
-unsigned int
-path_get_length (struct CadetPeerPath *path);
-
-/**
- * Mark path as invalid: keep it aroud for a while to avoid trying it in a loop.
- *
- * DHT_get sometimes returns bad cached results, for instance, on a locally
- * cached result where the PUT followed a path that is no longer current.
- *
- * @param p Path to invalidate.
- */
-void
-path_invalidate (struct CadetPeerPath *p);
-
-/**
- * Test if two paths are equivalent (equal or revese of each other).
- *
- * @param p1 First path
- * @param p2 Second path
- *
- * @return GNUNET_YES if both paths are equivalent
- *         GNUNET_NO otherwise
- */
-int
-path_equivalent (const struct CadetPeerPath *p1,
-                 const struct CadetPeerPath *p2);
-
-/**
- * Test if a path is valid (or at least not known to be invalid).
- *
- * @param path Path to test.
- *
- * @return #GNUNET_YES If the path is valid or unknown,
- *         #GNUNET_NO If the path is known to be invalid.
- */
-int
-path_is_valid (const struct CadetPeerPath *path);
-
-/**
- * Destroy the path and free any allocated resources linked to it
- *
- * @param p the path to destroy
- *
- * @return GNUNET_OK on success
- */
-int
-path_destroy (struct CadetPeerPath *p);
-
-/**
- * Compare two paths.
- *
- * @param p1 First path.
- * @param p2 Second path.
- *
- * @return > 0 if p1 is longer, or the first differing PEER_Id is higher on p1.
- *         < 0 if p2 is longer, or the first differing PEER_Id is higher on p2.
- *         0 if they are identical.
- */
-int
-path_cmp (const struct CadetPeerPath *p1, const struct CadetPeerPath *p2);
-
-/**
- * Builds a path from a PeerIdentity array.
- *
- * @param peers PeerIdentity array.
- * @param size Size of the @c peers array.
- * @param myid ID of local peer, to find @c own_pos.
- * @param own_pos Output parameter: own position in the path.
- *
- * @return Fixed and shortened path.
- */
-struct CadetPeerPath *
-path_build_from_peer_ids (struct GNUNET_PeerIdentity *peers,
-                          unsigned int size,
-                          GNUNET_PEER_Id myid,
-                          unsigned int *own_pos);
-
-/**
- * Path -> allocated one line string. Caller must free.
- *
- * @param p Path.
- */
-char *
-path_2s (struct CadetPeerPath *p);
-
-/**
- * Print info about the path for debug.
- *
- * @param p Path to debug.
- */
-void
-path_debug (struct CadetPeerPath *p);
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-  #endif
-  #ifdef __cplusplus
-}
-#endif
-
-
-/* ifndef CADET_PATH_H */
-#endif
diff --git a/src/cadet/cadet_protocol.h b/src/cadet/cadet_protocol.h
index d2426addb..560c186cd 100644
--- a/src/cadet/cadet_protocol.h
+++ b/src/cadet/cadet_protocol.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2001 - 2011 GNUnet e.V.
+     Copyright (C) 2007 - 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -19,8 +19,10 @@
 */
 
 /**
- * @author Bartlomiej Polot
  * @file cadet/cadet_protocol.h
+ * @brief P2P messages used by CADET
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
  */
 
 #ifndef CADET_PROTOCOL_H_
@@ -298,17 +300,10 @@ struct GNUNET_CADET_TunnelEncryptedMessage
    */
   struct GNUNET_MessageHeader header;
 
-#if NEW_CADET
   /**
    * Reserved, for alignment.
    */
   uint32_t reserved GNUNET_PACKED;
-#else
-  /**
-   * Maximum packet ID authorized.
-   */
-  struct CadetEncryptedMessageIdentifier cemi;
-#endif
 
   /**
    * ID of the connection.
@@ -322,89 +317,18 @@ struct GNUNET_CADET_TunnelEncryptedMessage
    */
   struct GNUNET_ShortHashCode hmac;
 
-  #if NEW_CADET
   /**
    * Axolotl-header that specifies which keys to use in which ratchet
    * to decrypt the body that follows.
    */
   struct GNUNET_CADET_AxHeader ax_header;
-#else
-  /**
-   * Number of messages sent with the current ratchet key.
-   */
-  uint32_t Ns GNUNET_PACKED;
-
-  /**
-   * Number of messages sent with the previous ratchet key.
-   */
-  uint32_t PNs GNUNET_PACKED;
 
   /**
-   * Current ratchet key.
-   */
-  struct GNUNET_CRYPTO_EcdhePublicKey DHRs;
-#endif
-  /**
    * Encrypted content follows.
    */
 };
 
 
-#ifndef NEW_CADET
-
-/**
- * Message to query a peer about its Flow Control status regarding a tunnel.
- *
- * It is NOT yet clear if we need this.
- */
-struct GNUNET_CADET_ConnectionHopByHopPollMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Last packet sent.
-   */
-  struct CadetEncryptedMessageIdentifier cemi;
-
-  /**
-   * ID of the connection.
-   */
-  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
-
-};
-
-
-/**
- * Message to acknowledge cadet encrypted traffic, used for
- * flow-control on a hop-by-hop basis on the connection-level. Note
- * that we do use the @e cemi from the tunnel layer as the connection
- * layer's header is included/shared with the tunnel layer messages,
- * and we only do flow control for the payload.
- */
-struct GNUNET_CADET_ConnectionEncryptedAckMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Maximum packet ID authorized.
-   */
-  struct CadetEncryptedMessageIdentifier cemi_max;
-
-  /**
-   * ID of the connection.
-   */
-  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
-};
-
-#endif
-
-
 /******************************************************************************/
 /*******************************   CHANNEL  ***********************************/
 /******************************************************************************/
@@ -450,83 +374,19 @@ struct GNUNET_CADET_ChannelManageMessage
    */
   struct GNUNET_MessageHeader header;
 
-#ifdef NEW_CADET
   /**
    * For alignment.
    */
   uint32_t reserved GNUNET_PACKED;
-#endif
-
-  /**
-   * ID of the channel
-   */
-  struct GNUNET_CADET_ChannelTunnelNumber ctn;
-};
-
-
-#ifndef NEW_CADET
-
-/**
- * Message for cadet data traffic.
- */
-struct GNUNET_CADET_ChannelAppDataMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_UNICAST,
-   *       #GNUNET_MESSAGE_TYPE_CADET_TO_ORIGIN
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * Unique ID of the payload message
-   */
-  /* NEW: struct ChannelMessageIdentifier */
-  uint32_t mid GNUNET_PACKED;
 
   /**
    * ID of the channel
    */
   struct GNUNET_CADET_ChannelTunnelNumber ctn;
-
-  /**
-   * Payload follows
-   */
 };
 
 
 /**
- * Message to acknowledge end-to-end data.
- */
-struct GNUNET_CADET_ChannelDataAckMessage
-{
-  /**
-   * Type: #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK
-   */
-  struct GNUNET_MessageHeader header;
-
-  /**
-   * ID of the channel
-   */
-  struct GNUNET_CADET_ChannelTunnelNumber ctn;
-
-  /**
-   * Bitfield of already-received messages past @e mid.
-   * pid +  1 @ LSB
-   * pid + 64 @ MSB
-   */
-  uint64_t futures GNUNET_PACKED;
-
-  /**
-   * Last message ID received.
-   */
-  /* NEW: struct ChannelMessageIdentifier */
-  uint32_t mid GNUNET_PACKED;
-};
-
-#else
-
-
-/**
  * Number used to uniquely identify messages in a CADET Channel.
  */
 struct ChannelMessageIdentifier
@@ -595,8 +455,6 @@ struct GNUNET_CADET_ChannelDataAckMessage
 };
 
 
-#endif
-
 GNUNET_NETWORK_STRUCT_END
 
 #if 0                           /* keep Emacsens' auto-indent happy */
diff --git a/src/cadet/cadet_test_lib.c b/src/cadet/cadet_test_lib.c
index 9a70dad49..1df6bff0d 100644
--- a/src/cadet/cadet_test_lib.c
+++ b/src/cadet/cadet_test_lib.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2012 GNUnet e.V.
+     Copyright (C) 2012, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -27,6 +27,7 @@
 #include "cadet_test_lib.h"
 #include "gnunet_cadet_service.h"
 
+
 /**
  * Test context for a CADET Test.
  */
@@ -40,7 +41,7 @@ struct GNUNET_CADET_TEST_Context
   /**
    * Array of handles to the CADET for each peer.
    */
-  struct GNUNET_CADET_Handle **cadetes;
+  struct GNUNET_CADET_Handle **cadets;
 
   /**
    * Operation associated with the connection to the CADET.
@@ -48,6 +49,11 @@ struct GNUNET_CADET_TEST_Context
   struct GNUNET_TESTBED_Operation **ops;
 
   /**
+   * Number of peers running, size of the arrays above.
+   */
+  unsigned int num_peers;
+
+  /**
    * Main function of the test to run once all CADETs are available.
    */
   GNUNET_CADET_TEST_AppMain app_main;
@@ -58,30 +64,35 @@ struct GNUNET_CADET_TEST_Context
   void *app_main_cls;
 
   /**
-   * Number of peers running, size of the arrays above.
+   * Handler for incoming tunnels.
    */
-  unsigned int num_peers;
+  GNUNET_CADET_ConnectEventHandler connects;
 
   /**
-   * Handler for incoming tunnels.
+   * Function called when the transmit window size changes.
    */
-  GNUNET_CADET_InboundChannelNotificationHandler *new_channel;
+  GNUNET_CADET_WindowSizeEventHandler window_changes;
 
   /**
    * Cleaner for destroyed incoming tunnels.
    */
-  GNUNET_CADET_ChannelEndHandler *cleaner;
+  GNUNET_CADET_DisconnectEventHandler disconnects;
 
   /**
    * Message handlers.
    */
-  struct GNUNET_CADET_MessageHandler* handlers;
+  struct GNUNET_MQ_MessageHandler *handlers;
 
   /**
    * Application ports.
    */
   const struct GNUNET_HashCode **ports;
 
+  /**
+   * Number of ports in #ports.
+   */
+  unsigned int port_count;
+
 };
 
 
@@ -94,6 +105,11 @@ struct GNUNET_CADET_TEST_AdapterContext
    * Peer number for the particular peer.
    */
   unsigned int peer;
+
+  /**
+   * Port handlers for open ports.
+   */
+  struct GNUNET_CADET_Port **ports;
  
   /**
    * General context.
@@ -114,26 +130,28 @@ struct GNUNET_CADET_TEST_AdapterContext
  */
 static void *
 cadet_connect_adapter (void *cls,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg)
+                       const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_CADET_TEST_AdapterContext *actx = cls;
   struct GNUNET_CADET_TEST_Context *ctx = actx->ctx;
   struct GNUNET_CADET_Handle *h;
+  unsigned int i;
 
-  h = GNUNET_CADET_connect (cfg,
-                           (void *) (long) actx->peer,
-                           ctx->cleaner,
-                           ctx->handlers);
+  h = GNUNET_CADET_connect (cfg);
   if (NULL == ctx->ports)
     return h;
 
-  for (int i = 0; NULL != ctx->ports[i]; i++)
+  actx->ports = GNUNET_new_array (ctx->port_count, struct GNUNET_CADET_Port *);
+  for (i = 0; i < ctx->port_count; i++)
   {
-    (void ) GNUNET_CADET_open_port (h, ctx->ports[i],
-                                    ctx->new_channel,
-                                    (void *) (long) actx->peer);
+    actx->ports[i] = GNUNET_CADET_open_port (h,
+                                             ctx->ports[i],
+                                             ctx->connects,
+                                             (void *) (long) actx->peer,
+                                             ctx->window_changes,
+                                             ctx->disconnects,
+                                             ctx->handlers);
   }
-
   return h;
 }
 
@@ -152,6 +170,15 @@ cadet_disconnect_adapter (void *cls,
   struct GNUNET_CADET_Handle *cadet = op_result;
   struct GNUNET_CADET_TEST_AdapterContext *actx = cls;
 
+  if (NULL != actx->ports)
+  {
+    for (int i = 0; i < actx->ctx->port_count; i++)
+    {
+      GNUNET_CADET_close_port (actx->ports[i]);
+      actx->ports[i] = NULL;
+    }
+    GNUNET_free (actx->ports);
+  }
   GNUNET_free (actx);
   GNUNET_CADET_disconnect (cadet);
 }
@@ -186,18 +213,18 @@ cadet_connect_cb (void *cls,
   for (i = 0; i < ctx->num_peers; i++)
     if (op == ctx->ops[i])
     {
-      ctx->cadetes[i] = ca_result;
+      ctx->cadets[i] = ca_result;
       GNUNET_log (GNUNET_ERROR_TYPE_INFO, "...cadet %u connected\n", i);
     }
   for (i = 0; i < ctx->num_peers; i++)
-    if (NULL == ctx->cadetes[i])
+    if (NULL == ctx->cadets[i])
       return; /* still some CADET connections missing */
   /* all CADET connections ready! */
   ctx->app_main (ctx->app_main_cls,
                  ctx,
                  ctx->num_peers,
                  ctx->peers,
-                 ctx->cadetes);
+                 ctx->cadets);
 }
 
 
@@ -213,7 +240,7 @@ GNUNET_CADET_TEST_cleanup (struct GNUNET_CADET_TEST_Context *ctx)
     ctx->ops[i] = NULL;
   }
   GNUNET_free (ctx->ops);
-  GNUNET_free (ctx->cadetes);
+  GNUNET_free (ctx->cadets);
   GNUNET_free (ctx);
   GNUNET_SCHEDULER_shutdown ();
 }
@@ -243,12 +270,23 @@ cadet_test_run (void *cls,
   struct GNUNET_CADET_TEST_Context *ctx = cls;
   unsigned int i;
 
+  if (0 != links_failed)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Some links failed (%u), ending\n",
+                links_failed);
+    exit (2);
+  }
+
   if  (num_peers != ctx->num_peers)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Peers started %u/%u, ending\n",
                 num_peers, ctx->num_peers);
     exit (1);
   }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Testbed up, %u peers and %u links\n",
+              num_peers, links_succeeded);
   ctx->peers = peers;
   for (i = 0; i < num_peers; i++)
   {
@@ -270,31 +308,52 @@ cadet_test_run (void *cls,
 }
 
 
+/**
+ * Run a test using the given name, configuration file and number of peers.
+ * All cadet callbacks will receive the peer number (long) as the closure.
+ *
+ * @param testname Name of the test (for logging).
+ * @param cfgfile Name of the configuration file.
+ * @param num_peers Number of peers to start.
+ * @param tmain Main function to run once the testbed is ready.
+ * @param tmain_cls Closure for @a tmain.
+ * @param connects Handler for incoming channels.
+ * @param window_changes Handler for the window size change notification.
+ * @param disconnects Cleaner for destroyed incoming channels.
+ * @param handlers Message handlers.
+ * @param ports Ports the peers offer, NULL-terminated.
+ */
 void
-GNUNET_CADET_TEST_run (const char *testname,
-                      const char *cfgname,
-                      unsigned int num_peers,
-                      GNUNET_CADET_TEST_AppMain tmain,
-                      void *tmain_cls,
-                      GNUNET_CADET_InboundChannelNotificationHandler new_channel,
-                      GNUNET_CADET_ChannelEndHandler cleaner,
-                      struct GNUNET_CADET_MessageHandler* handlers,
-                      const struct GNUNET_HashCode **ports)
+GNUNET_CADET_TEST_ruN (const char *testname,
+                       const char *cfgfile,
+                       unsigned int num_peers,
+                       GNUNET_CADET_TEST_AppMain tmain,
+                       void *tmain_cls,
+                       GNUNET_CADET_ConnectEventHandler connects,
+                       GNUNET_CADET_WindowSizeEventHandler window_changes,
+                       GNUNET_CADET_DisconnectEventHandler disconnects,
+                       struct GNUNET_MQ_MessageHandler *handlers,
+                       const struct GNUNET_HashCode **ports)
 {
   struct GNUNET_CADET_TEST_Context *ctx;
 
   ctx = GNUNET_new (struct GNUNET_CADET_TEST_Context);
   ctx->num_peers = num_peers;
-  ctx->ops = GNUNET_malloc (num_peers * sizeof (struct GNUNET_TESTBED_Operation *));
-  ctx->cadetes = GNUNET_malloc (num_peers * sizeof (struct GNUNET_CADET_Handle *));
+  ctx->ops = GNUNET_new_array (num_peers, struct GNUNET_TESTBED_Operation *);
+  ctx->cadets = GNUNET_new_array (num_peers, struct GNUNET_CADET_Handle *);
   ctx->app_main = tmain;
   ctx->app_main_cls = tmain_cls;
-  ctx->new_channel = new_channel;
-  ctx->cleaner = cleaner;
-  ctx->handlers = handlers;
+  ctx->connects = connects;
+  ctx->window_changes = window_changes;
+  ctx->disconnects = disconnects;
+  ctx->handlers = GNUNET_MQ_copy_handlers (handlers);
   ctx->ports = ports;
+  ctx->port_count = 0;
+  while (NULL != ctx->ports[ctx->port_count])
+    ctx->port_count++;
+
   GNUNET_TESTBED_test_run (testname,
-                           cfgname,
+                           cfgfile,
                            num_peers,
                            0LL, NULL, NULL,
                            &cadet_test_run, ctx);
diff --git a/src/cadet/cadet_test_lib.h b/src/cadet/cadet_test_lib.h
index 464977d42..4b3a6b18d 100644
--- a/src/cadet/cadet_test_lib.h
+++ b/src/cadet/cadet_test_lib.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2012 GNUnet e.V.
+     Copyright (C) 2012,2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -49,41 +49,41 @@ struct GNUNET_CADET_TEST_Context;
  * @param ctx Argument to give to GNUNET_CADET_TEST_cleanup on test end.
  * @param num_peers Number of peers that are running.
  * @param peers Array of peers.
- * @param cadetes Handle to each of the CADETs of the peers.
+ * @param cadets Handle to each of the CADETs of the peers.
  */
 typedef void (*GNUNET_CADET_TEST_AppMain) (void *cls,
                                           struct GNUNET_CADET_TEST_Context *ctx,
                                           unsigned int num_peers,
                                           struct GNUNET_TESTBED_Peer **peers,
-                                          struct GNUNET_CADET_Handle **cadetes);
+                                          struct GNUNET_CADET_Handle **cadets);
 
 
 /**
- * Run a test using the given name, configuration file and number of
- * peers.
- * All cadet callbacks will receive the peer number as the closure.
+ * Run a test using the given name, configuration file and number of peers.
+ * All cadet callbacks will receive the peer number (long) as the closure.
  *
  * @param testname Name of the test (for logging).
- * @param cfgname Name of the configuration file.
+ * @param cfgfile Name of the configuration file.
  * @param num_peers Number of peers to start.
  * @param tmain Main function to run once the testbed is ready.
- * @param tmain_cls Closure for 'tmain'.
- * @param new_channel Handler for incoming tunnels.
- * @param cleaner Cleaner for destroyed incoming tunnels.
+ * @param tmain_cls Closure for @a tmain.
+ * @param connects Handler for incoming channels.
+ * @param window_changes Handler for the window size change notification.
+ * @param disconnects Cleaner for destroyed incoming channels.
  * @param handlers Message handlers.
  * @param ports Ports the peers offer, NULL-terminated.
  */
 void
-GNUNET_CADET_TEST_run (const char *testname,
-                      const char *cfgname,
-                      unsigned int num_peers,
-                      GNUNET_CADET_TEST_AppMain tmain,
-                      void *tmain_cls,
-                      GNUNET_CADET_InboundChannelNotificationHandler new_channel,
-                      GNUNET_CADET_ChannelEndHandler cleaner,
-                      struct GNUNET_CADET_MessageHandler* handlers,
-                      const struct GNUNET_HashCode **ports);
-
+GNUNET_CADET_TEST_ruN (const char *testname,
+                       const char *cfgfile,
+                       unsigned int num_peers,
+                       GNUNET_CADET_TEST_AppMain tmain,
+                       void *tmain_cls,
+                       GNUNET_CADET_ConnectEventHandler connects,
+                       GNUNET_CADET_WindowSizeEventHandler window_changes,
+                       GNUNET_CADET_DisconnectEventHandler disconnects,
+                       struct GNUNET_MQ_MessageHandler *handlers,
+                       const struct GNUNET_HashCode **ports);
 
 /**
  * Clean up the testbed.
diff --git a/src/cadet/cadet_test_lib_new.c b/src/cadet/cadet_test_lib_new.c
deleted file mode 100644
index c3a1540f4..000000000
--- a/src/cadet/cadet_test_lib_new.c
+++ /dev/null
@@ -1,362 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2012, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/cadet_test_lib.c
- * @author Bartlomiej Polot
- * @brief library for writing CADET tests
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "cadet_test_lib_new.h"
-#include "gnunet_cadet_service.h"
-
-
-/**
- * Test context for a CADET Test.
- */
-struct GNUNET_CADET_TEST_Context
-{
-  /**
-   * Array of running peers.
-   */
-  struct GNUNET_TESTBED_Peer **peers;
-
-  /**
-   * Array of handles to the CADET for each peer.
-   */
-  struct GNUNET_CADET_Handle **cadets;
-
-  /**
-   * Operation associated with the connection to the CADET.
-   */
-  struct GNUNET_TESTBED_Operation **ops;
-
-  /**
-   * Number of peers running, size of the arrays above.
-   */
-  unsigned int num_peers;
-
-  /**
-   * Main function of the test to run once all CADETs are available.
-   */
-  GNUNET_CADET_TEST_AppMain app_main;
-
-  /**
-   * Closure for 'app_main'.
-   */
-  void *app_main_cls;
-
-  /**
-   * Handler for incoming tunnels.
-   */
-  GNUNET_CADET_ConnectEventHandler connects;
-
-  /**
-   * Function called when the transmit window size changes.
-   */
-  GNUNET_CADET_WindowSizeEventHandler window_changes;
-
-  /**
-   * Cleaner for destroyed incoming tunnels.
-   */
-  GNUNET_CADET_DisconnectEventHandler disconnects;
-
-  /**
-   * Message handlers.
-   */
-  struct GNUNET_MQ_MessageHandler *handlers;
-
-  /**
-   * Application ports.
-   */
-  const struct GNUNET_HashCode **ports;
-
-  /**
-   * Number of ports in #ports.
-   */
-  unsigned int port_count;
-
-};
-
-
-/**
- * Context for a cadet adapter callback.
- */
-struct GNUNET_CADET_TEST_AdapterContext
-{
-  /**
-   * Peer number for the particular peer.
-   */
-  unsigned int peer;
-
-  /**
-   * Port handlers for open ports.
-   */
-  struct GNUNET_CADET_Port **ports;
- 
-  /**
-   * General context.
-   */
-  struct GNUNET_CADET_TEST_Context *ctx;
-};
-
-
-/**
- * Adapter function called to establish a connection to
- * the CADET service.
- *
- * @param cls closure
- * @param cfg configuration of the peer to connect to; will be available until
- *          GNUNET_TESTBED_operation_done() is called on the operation returned
- *          from GNUNET_TESTBED_service_connect()
- * @return service handle to return in 'op_result', NULL on error
- */
-static void *
-cadet_connect_adapter (void *cls,
-                       const struct GNUNET_CONFIGURATION_Handle *cfg)
-{
-  struct GNUNET_CADET_TEST_AdapterContext *actx = cls;
-  struct GNUNET_CADET_TEST_Context *ctx = actx->ctx;
-  struct GNUNET_CADET_Handle *h;
-  unsigned int i;
-
-  h = GNUNET_CADET_connecT (cfg);
-  if (NULL == ctx->ports)
-    return h;
-
-  actx->ports = GNUNET_new_array (ctx->port_count, struct GNUNET_CADET_Port *);
-  for (i = 0; i < ctx->port_count; i++)
-  {
-    actx->ports[i] = GNUNET_CADET_open_porT (h,
-                                             ctx->ports[i],
-                                             ctx->connects,
-                                             (void *) (long) actx->peer,
-                                             ctx->window_changes,
-                                             ctx->disconnects,
-                                             ctx->handlers);
-  }
-  return h;
-}
-
-
-/**
- * Adapter function called to destroy a connection to
- * the CADET service.
- *
- * @param cls closure
- * @param op_result service handle returned from the connect adapter
- */
-static void
-cadet_disconnect_adapter (void *cls,
-                         void *op_result)
-{
-  struct GNUNET_CADET_Handle *cadet = op_result;
-  struct GNUNET_CADET_TEST_AdapterContext *actx = cls;
-
-  if (NULL != actx->ports)
-  {
-    for (int i = 0; i < actx->ctx->port_count; i++)
-    {
-      GNUNET_CADET_close_port (actx->ports[i]);
-      actx->ports[i] = NULL;
-    }
-    GNUNET_free (actx->ports);
-  }
-  GNUNET_free (actx);
-  GNUNET_CADET_disconnect (cadet);
-}
-
-
-/**
- * Callback to be called when a service connect operation is completed.
- *
- * @param cls The callback closure from functions generating an operation.
- * @param op The operation that has been finished.
- * @param ca_result The service handle returned from
- *                  GNUNET_TESTBED_ConnectAdapter() (cadet handle).
- * @param emsg Error message in case the operation has failed.
- *             NULL if operation has executed successfully.
- */
-static void
-cadet_connect_cb (void *cls,
-                 struct GNUNET_TESTBED_Operation *op,
-                 void *ca_result,
-                 const char *emsg)
-{
-  struct GNUNET_CADET_TEST_Context *ctx = cls;
-  unsigned int i;
-
-  if (NULL != emsg)
-  {
-    fprintf (stderr, "Failed to connect to CADET service: %s\n",
-             emsg);
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  for (i = 0; i < ctx->num_peers; i++)
-    if (op == ctx->ops[i])
-    {
-      ctx->cadets[i] = ca_result;
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO, "...cadet %u connected\n", i);
-    }
-  for (i = 0; i < ctx->num_peers; i++)
-    if (NULL == ctx->cadets[i])
-      return; /* still some CADET connections missing */
-  /* all CADET connections ready! */
-  ctx->app_main (ctx->app_main_cls,
-                 ctx,
-                 ctx->num_peers,
-                 ctx->peers,
-                 ctx->cadets);
-}
-
-
-void
-GNUNET_CADET_TEST_cleanup (struct GNUNET_CADET_TEST_Context *ctx)
-{
-  unsigned int i;
-
-  for (i = 0; i < ctx->num_peers; i++)
-  {
-    GNUNET_assert (NULL != ctx->ops[i]);
-    GNUNET_TESTBED_operation_done (ctx->ops[i]);
-    ctx->ops[i] = NULL;
-  }
-  GNUNET_free (ctx->ops);
-  GNUNET_free (ctx->cadets);
-  GNUNET_free (ctx);
-  GNUNET_SCHEDULER_shutdown ();
-}
-
-
-/**
- * Callback run when the testbed is ready (peers running and connected to
- * each other)
- *
- * @param cls Closure (context).
- * @param h the run handle
- * @param num_peers Number of peers that are running.
- * @param peers Handles to each one of the @c num_peers peers.
- * @param links_succeeded the number of overlay link connection attempts that
- *          succeeded
- * @param links_failed the number of overlay link connection attempts that
- *          failed
- */
-static void
-cadet_test_run (void *cls,
-               struct GNUNET_TESTBED_RunHandle *h,
-               unsigned int num_peers,
-               struct GNUNET_TESTBED_Peer **peers,
-               unsigned int links_succeeded,
-               unsigned int links_failed)
-{
-  struct GNUNET_CADET_TEST_Context *ctx = cls;
-  unsigned int i;
-
-  if (0 != links_failed)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Some links failed (%u), ending\n",
-                links_failed);
-    exit (2);
-  }
-
-  if  (num_peers != ctx->num_peers)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Peers started %u/%u, ending\n",
-                num_peers, ctx->num_peers);
-    exit (1);
-  }
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Testbed up, %u peers and %u links\n",
-              num_peers, links_succeeded);
-  ctx->peers = peers;
-  for (i = 0; i < num_peers; i++)
-  {
-    struct GNUNET_CADET_TEST_AdapterContext *newctx;
-    newctx = GNUNET_new (struct GNUNET_CADET_TEST_AdapterContext);
-    newctx->peer = i;
-    newctx->ctx = ctx;
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Connecting to cadet %u\n", i);
-    ctx->ops[i] = GNUNET_TESTBED_service_connect (ctx,
-                                                  peers[i],
-                                                  "cadet",
-                                                  &cadet_connect_cb,
-                                                  ctx,
-                                                  &cadet_connect_adapter,
-                                                  &cadet_disconnect_adapter,
-                                                  newctx);
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "op handle %p\n", ctx->ops[i]);
-  }
-}
-
-
-/**
- * Run a test using the given name, configuration file and number of peers.
- * All cadet callbacks will receive the peer number (long) as the closure.
- *
- * @param testname Name of the test (for logging).
- * @param cfgfile Name of the configuration file.
- * @param num_peers Number of peers to start.
- * @param tmain Main function to run once the testbed is ready.
- * @param tmain_cls Closure for @a tmain.
- * @param connects Handler for incoming channels.
- * @param window_changes Handler for the window size change notification.
- * @param disconnects Cleaner for destroyed incoming channels.
- * @param handlers Message handlers.
- * @param ports Ports the peers offer, NULL-terminated.
- */
-void
-GNUNET_CADET_TEST_ruN (const char *testname,
-                       const char *cfgfile,
-                       unsigned int num_peers,
-                       GNUNET_CADET_TEST_AppMain tmain,
-                       void *tmain_cls,
-                       GNUNET_CADET_ConnectEventHandler connects,
-                       GNUNET_CADET_WindowSizeEventHandler window_changes,
-                       GNUNET_CADET_DisconnectEventHandler disconnects,
-                       struct GNUNET_MQ_MessageHandler *handlers,
-                       const struct GNUNET_HashCode **ports)
-{
-  struct GNUNET_CADET_TEST_Context *ctx;
-
-  ctx = GNUNET_new (struct GNUNET_CADET_TEST_Context);
-  ctx->num_peers = num_peers;
-  ctx->ops = GNUNET_new_array (num_peers, struct GNUNET_TESTBED_Operation *);
-  ctx->cadets = GNUNET_new_array (num_peers, struct GNUNET_CADET_Handle *);
-  ctx->app_main = tmain;
-  ctx->app_main_cls = tmain_cls;
-  ctx->connects = connects;
-  ctx->window_changes = window_changes;
-  ctx->disconnects = disconnects;
-  ctx->handlers = GNUNET_MQ_copy_handlers (handlers);
-  ctx->ports = ports;
-  ctx->port_count = 0;
-  while (NULL != ctx->ports[ctx->port_count])
-    ctx->port_count++;
-
-  GNUNET_TESTBED_test_run (testname,
-                           cfgfile,
-                           num_peers,
-                           0LL, NULL, NULL,
-                           &cadet_test_run, ctx);
-}
-
-/* end of cadet_test_lib.c */
diff --git a/src/cadet/cadet_test_lib_new.h b/src/cadet/cadet_test_lib_new.h
deleted file mode 100644
index 4b3a6b18d..000000000
--- a/src/cadet/cadet_test_lib_new.h
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2012,2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/cadet_test_lib.h
- * @author Bartlomiej Polot
- * @brief library for writing CADET tests
- */
-#ifndef CADET_TEST_LIB_H
-#define CADET_TEST_LIB_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "gnunet_testbed_service.h"
-#include "gnunet_cadet_service.h"
-
-/**
- * Test context for a CADET Test.
- */
-struct GNUNET_CADET_TEST_Context;
-
-
-/**
- * Main function of a CADET test.
- *
- * @param cls Closure.
- * @param ctx Argument to give to GNUNET_CADET_TEST_cleanup on test end.
- * @param num_peers Number of peers that are running.
- * @param peers Array of peers.
- * @param cadets Handle to each of the CADETs of the peers.
- */
-typedef void (*GNUNET_CADET_TEST_AppMain) (void *cls,
-                                          struct GNUNET_CADET_TEST_Context *ctx,
-                                          unsigned int num_peers,
-                                          struct GNUNET_TESTBED_Peer **peers,
-                                          struct GNUNET_CADET_Handle **cadets);
-
-
-/**
- * Run a test using the given name, configuration file and number of peers.
- * All cadet callbacks will receive the peer number (long) as the closure.
- *
- * @param testname Name of the test (for logging).
- * @param cfgfile Name of the configuration file.
- * @param num_peers Number of peers to start.
- * @param tmain Main function to run once the testbed is ready.
- * @param tmain_cls Closure for @a tmain.
- * @param connects Handler for incoming channels.
- * @param window_changes Handler for the window size change notification.
- * @param disconnects Cleaner for destroyed incoming channels.
- * @param handlers Message handlers.
- * @param ports Ports the peers offer, NULL-terminated.
- */
-void
-GNUNET_CADET_TEST_ruN (const char *testname,
-                       const char *cfgfile,
-                       unsigned int num_peers,
-                       GNUNET_CADET_TEST_AppMain tmain,
-                       void *tmain_cls,
-                       GNUNET_CADET_ConnectEventHandler connects,
-                       GNUNET_CADET_WindowSizeEventHandler window_changes,
-                       GNUNET_CADET_DisconnectEventHandler disconnects,
-                       struct GNUNET_MQ_MessageHandler *handlers,
-                       const struct GNUNET_HashCode **ports);
-
-/**
- * Clean up the testbed.
- *
- * @param ctx handle for the testbed
- */
-void
-GNUNET_CADET_TEST_cleanup (struct GNUNET_CADET_TEST_Context *ctx);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-
-/* ifndef CADET_TEST_LIB_H */
-#endif
diff --git a/src/cadet/gnunet-cadet.c b/src/cadet/gnunet-cadet.c
index c194a5bc1..57eeac735 100644
--- a/src/cadet/gnunet-cadet.c
+++ b/src/cadet/gnunet-cadet.c
@@ -772,15 +772,15 @@ run (void *cls,
        && target_id != NULL)
   {
     FPRINTF (stderr,
-             _("You must NOT give a TARGET "
-               "when using 'request all' options\n"));
+             _("Extra arguments are not applicable "
+               "in combination with this option.\n"));
     return;
   }
 
   if (GNUNET_YES == dump)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "requesting debug dump\n");
+                "Requesting debug dump\n");
     job = GNUNET_SCHEDULER_add_now (&request_dump,
                                     NULL);
   }
@@ -829,7 +829,7 @@ run (void *cls,
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Connecting to CADET service\n");
-  mh = GNUNET_CADET_connecT (cfg);
+  mh = GNUNET_CADET_connect (cfg);
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
                                  NULL);
   if (NULL == mh)
@@ -844,7 +844,7 @@ run (void *cls,
     GNUNET_CRYPTO_hash (listen_port,
                         strlen (listen_port),
                         &porthash);
-    lp = GNUNET_CADET_open_porT (mh,
+    lp = GNUNET_CADET_open_port (mh,
                                  &porthash,
                                  &channel_incoming,
                                  NULL,
@@ -876,7 +876,7 @@ run (void *cls,
     GNUNET_CRYPTO_hash (target_port,
                         strlen(target_port),
                         &porthash);
-    ch = GNUNET_CADET_channel_creatE (mh,
+    ch = GNUNET_CADET_channel_create (mh,
                                       NULL,
                                       &pid,
                                       &porthash,
@@ -919,32 +919,55 @@ main (int argc,
       char *const *argv)
 {
   int res;
-  const char helpstr[] = "Create channels and retreive info about cadets status.";
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'C', "connection", "CONNECTION_ID",
-     gettext_noop ("provide information about a particular connection"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &conn_id},
-    {'e', "echo", NULL,
-     gettext_noop ("activate echo mode"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &echo},
-    {'d', "dump", NULL,
-     gettext_noop ("dump debug information to STDERR"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &dump},
-    {'o', "open-port", "PORT",
-     gettext_noop ("port to listen to"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &listen_port},
-    {'p', "peer", "PEER_ID",
-     gettext_noop ("provide information about a patricular peer"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &peer_id},
-    {'P', "peers", NULL,
-      gettext_noop ("provide information about all peers"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &request_peers},
-    {'t', "tunnel", "TUNNEL_ID",
-     gettext_noop ("provide information about a particular tunnel"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &tunnel_id},
-    {'T', "tunnels", NULL,
-     gettext_noop ("provide information about all tunnels"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &request_tunnels},
+  const char helpstr[] = "Create tunnels and retrieve info about CADET's status.";
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    /* I would use the terminology 'circuit' here...  --lynX */
+    GNUNET_GETOPT_OPTION_STRING ('C',
+                                 "connection",
+                                 "CONNECTION_ID",
+                                 gettext_noop ("Provide information about a particular connection"),
+                                 &conn_id),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('e',
+                                  "echo",
+                                  gettext_noop ("Activate echo mode"),
+                                  &echo), 
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('d',
+                                  "dump",
+                                  gettext_noop ("Dump debug information to STDERR"),
+                                  &dump),
+
+    GNUNET_GETOPT_OPTION_STRING ('o',
+                                 "open-port",
+                                 "SHARED_SECRET",
+                                 gettext_noop ("Listen for connections using a shared secret among sender and recipient"),
+                                 &listen_port),
+
+
+    GNUNET_GETOPT_OPTION_STRING ('p',
+                                 "peer",
+                                 "PEER_ID",
+                                 gettext_noop ("Provide information about a patricular peer"),
+                                 &peer_id),
+
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('P',
+                                  "peers",
+                                  gettext_noop ("Provide information about all peers"),
+                                  &request_peers),
+
+    GNUNET_GETOPT_OPTION_STRING ('t',
+                                 "tunnel",
+                                 "TUNNEL_ID",
+                                 gettext_noop ("Provide information about a particular tunnel"),
+                                 &tunnel_id),
+
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('T',
+                                  "tunnels",
+                                  gettext_noop ("Provide information about all tunnels"),
+                                  &request_tunnels),
 
     GNUNET_GETOPT_OPTION_END
   };
@@ -955,7 +978,7 @@ main (int argc,
     return 2;
 
   res = GNUNET_PROGRAM_run (argc, argv,
-                            "gnunet-cadet (OPTIONS | TARGET PORT)",
+                            "gnunet-cadet (OPTIONS | PEER_ID SHARED_SECRET)",
                             gettext_noop (helpstr),
                             options, &run, NULL);
 
diff --git a/src/cadet/gnunet-service-cadet-new.c b/src/cadet/gnunet-service-cadet-new.c
deleted file mode 100644
index de85db5b6..000000000
--- a/src/cadet/gnunet-service-cadet-new.c
+++ /dev/null
@@ -1,1490 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2013, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new.c
- * @brief GNUnet CADET service with encryption
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * Dictionary:
- * - peer: other cadet instance. If there is direct connection it's a neighbor.
- * - path: series of directly connected peer from one peer to another.
- * - connection: path which is being used in a tunnel.
- * - tunnel: encrypted connection to a peer, neighbor or not.
- * - channel: logical link between two clients, on the same or different peers.
- *            have properties like reliability.
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "cadet.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_channel.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_core.h"
-#include "gnunet-service-cadet-new_dht.h"
-#include "gnunet-service-cadet-new_hello.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
-
-#define LOG(level, ...) GNUNET_log (level,__VA_ARGS__)
-
-
-/**
- * Struct containing information about a client of the service
- */
-struct CadetClient
-{
-  /**
-   * Linked list next
-   */
-  struct CadetClient *next;
-
-  /**
-   * Linked list prev
-   */
-  struct CadetClient *prev;
-
-  /**
-   * Tunnels that belong to this client, indexed by local id,
-   * value is a `struct CadetChannel`.
-   */
-  struct GNUNET_CONTAINER_MultiHashMap32 *channels;
-
-  /**
-   * Handle to communicate with the client
-   */
-  struct GNUNET_MQ_Handle *mq;
-
-  /**
-   * Client handle.
-   */
-  struct GNUNET_SERVICE_Client *client;
-
-  /**
-   * Ports that this client has declared interest in.
-   * Indexed by port, contains *Client.
-   */
-  struct GNUNET_CONTAINER_MultiHashMap *ports;
-
-  /**
-   * Channel ID to use for the next incoming channel for this client.
-   * Wraps around (in theory).
-   */
-  struct GNUNET_CADET_ClientChannelNumber next_ccn;
-
-  /**
-   * ID of the client, mainly for debug messages. Purely internal to this file.
-   */
-  unsigned int id;
-};
-
-/******************************************************************************/
-/***********************      GLOBAL VARIABLES     ****************************/
-/******************************************************************************/
-
-/****************************** Global variables ******************************/
-
-/**
- * Handle to our configuration.
- */
-const struct GNUNET_CONFIGURATION_Handle *cfg;
-
-/**
- * Handle to the statistics service.
- */
-struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Handle to communicate with ATS.
- */
-struct GNUNET_ATS_ConnectivityHandle *ats_ch;
-
-/**
- * Local peer own ID.
- */
-struct GNUNET_PeerIdentity my_full_id;
-
-/**
- * Own private key.
- */
-struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
-
-/**
- * Signal that shutdown is happening: prevent recovery measures.
- */
-int shutting_down;
-
-/**
- * DLL with all the clients, head.
- */
-static struct CadetClient *clients_head;
-
-/**
- * DLL with all the clients, tail.
- */
-static struct CadetClient *clients_tail;
-
-/**
- * Next ID to assign to a client.
- */
-static unsigned int next_client_id;
-
-/**
- * All ports clients of this peer have opened.
- */
-struct GNUNET_CONTAINER_MultiHashMap *open_ports;
-
-/**
- * Map from ports to channels where the ports were closed at the
- * time we got the inbound connection.
- * Indexed by port, contains `struct CadetChannel`.
- */
-struct GNUNET_CONTAINER_MultiHashMap *loose_channels;
-
-/**
- * Map from PIDs to `struct CadetPeer` entries.
- */
-struct GNUNET_CONTAINER_MultiPeerMap *peers;
-
-/**
- * Map from `struct GNUNET_CADET_ConnectionTunnelIdentifier`
- * hash codes to `struct CadetConnection` objects.
- */
-struct GNUNET_CONTAINER_MultiShortmap *connections;
-
-/**
- * How many messages are needed to trigger an AXOLOTL ratchet advance.
- */
-unsigned long long ratchet_messages;
-
-/**
- * How long until we trigger a ratched advance due to time.
- */
-struct GNUNET_TIME_Relative ratchet_time;
-
-/**
- * How frequently do we send KEEPALIVE messages on idle connections?
- */
-struct GNUNET_TIME_Relative keepalive_period;
-
-/**
- * Set to non-zero values to create random drops to test retransmissions.
- */
-unsigned long long drop_percent;
-
-
-/**
- * Send a message to a client.
- *
- * @param c client to get the message
- * @param env envelope with the message
- */
-void
-GSC_send_to_client (struct CadetClient *c,
-                    struct GNUNET_MQ_Envelope *env)
-{
-  GNUNET_MQ_send (c->mq,
-                  env);
-}
-
-
-/**
- * Return identifier for a client as a string.
- *
- * @param c client to identify
- * @return string for debugging
- */
-const char *
-GSC_2s (struct CadetClient *c)
-{
-  static char buf[32];
-
-  GNUNET_snprintf (buf,
-                   sizeof (buf),
-                   "Client(%u)",
-                   c->id);
-  return buf;
-}
-
-
-/**
- * Lookup channel of client @a c by @a ccn.
- *
- * @param c client to look in
- * @param ccn channel ID to look up
- * @return NULL if no such channel exists
- */
-static struct CadetChannel *
-lookup_channel (struct CadetClient *c,
-                struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  return GNUNET_CONTAINER_multihashmap32_get (c->channels,
-                                              ntohl (ccn.channel_of_client));
-}
-
-
-/**
- * Obtain the next LID to use for incoming connections to
- * the given client.
- *
- * @param c client handle
- */
-static struct GNUNET_CADET_ClientChannelNumber
-client_get_next_ccn (struct CadetClient *c)
-{
-  struct GNUNET_CADET_ClientChannelNumber ccn = c->next_ccn;
-
-  /* increment until we have a free one... */
-  while (NULL !=
-         lookup_channel (c,
-                         ccn))
-  {
-    ccn.channel_of_client
-      = htonl (1 + (ntohl (ccn.channel_of_client)));
-    if (ntohl (ccn.channel_of_client) >=
-        GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-      ccn.channel_of_client = htonl (0);
-  }
-  c->next_ccn.channel_of_client
-    = htonl (1 + (ntohl (ccn.channel_of_client)));
-  return ccn;
-}
-
-
-/**
- * Bind incoming channel to this client, and notify client about
- * incoming connection.  Caller is responsible for notifying the other
- * peer about our acceptance of the channel.
- *
- * @param c client to bind to
- * @param ch channel to be bound
- * @param dest peer that establishes the connection
- * @param port port number
- * @param options options
- * @return local channel number assigned to the new client
- */
-struct GNUNET_CADET_ClientChannelNumber
-GSC_bind (struct CadetClient *c,
-          struct CadetChannel *ch,
-          struct CadetPeer *dest,
-          const struct GNUNET_HashCode *port,
-          uint32_t options)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalChannelCreateMessage *cm;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  ccn = client_get_next_ccn (c);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_put (c->channels,
-                                                      ntohl (ccn.channel_of_client),
-                                                      ch,
-                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Accepting incoming %s from %s on open port %s (%u), assigning ccn %X\n",
-       GCCH_2s (ch),
-       GCP_2s (dest),
-       GNUNET_h2s (port),
-       (uint32_t) ntohl (options),
-       (uint32_t) ntohl (ccn.channel_of_client));
-  /* notify local client about incoming connection! */
-  env = GNUNET_MQ_msg (cm,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
-  cm->ccn = ccn;
-  cm->port = *port;
-  cm->opt = htonl (options);
-  cm->peer = *GCP_get_id (dest);
-  GSC_send_to_client (c,
-                      env);
-  return ccn;
-}
-
-
-/**
- * Callback invoked on all peers to destroy all tunnels
- * that may still exist.
- *
- * @param cls NULL
- * @param pid identify of a peer
- * @param value a `struct CadetPeer` that may still have a tunnel
- * @return #GNUNET_OK (iterate over all entries)
- */
-static int
-destroy_tunnels_now (void *cls,
-                     const struct GNUNET_PeerIdentity *pid,
-                     void *value)
-{
-  struct CadetPeer *cp = value;
-  struct CadetTunnel *t = GCP_get_tunnel (cp,
-                                          GNUNET_NO);
-
-  if (NULL != t)
-    GCT_destroy_tunnel_now (t);
-  return GNUNET_OK;
-}
-
-
-/**
- * Callback invoked on all peers to destroy all tunnels
- * that may still exist.
- *
- * @param cls NULL
- * @param pid identify of a peer
- * @param value a `struct CadetPeer` that may still have a tunnel
- * @return #GNUNET_OK (iterate over all entries)
- */
-static int
-destroy_paths_now (void *cls,
-                   const struct GNUNET_PeerIdentity *pid,
-                   void *value)
-{
-  struct CadetPeer *cp = value;
-
-  GCP_drop_owned_paths (cp);
-  return GNUNET_OK;
-}
-
-
-/**
- * Shutdown everything once the clients have disconnected.
- */
-static void
-shutdown_rest ()
-{
-  if (NULL != stats)
-  {
-    GNUNET_STATISTICS_destroy (stats,
-                               GNUNET_NO);
-    stats = NULL;
-  }
-  if (NULL != open_ports)
-  {
-    GNUNET_CONTAINER_multihashmap_destroy (open_ports);
-    open_ports = NULL;
-  }
-  if (NULL != loose_channels)
-  {
-    GNUNET_CONTAINER_multihashmap_destroy (loose_channels);
-    loose_channels = NULL;
-  }
-  /* Destroy tunnels.  Note that all channels must be destroyed first! */
-  GCP_iterate_all (&destroy_tunnels_now,
-                   NULL);
-  /* All tunnels, channels, connections and CORE must be down before this point. */
-  GCP_iterate_all (&destroy_paths_now,
-                   NULL);
-  /* All paths, tunnels, channels, connections and CORE must be down before this point. */
-  GCP_destroy_all_peers ();
-  if (NULL != peers)
-  {
-    GNUNET_CONTAINER_multipeermap_destroy (peers);
-    peers = NULL;
-  }
-  if (NULL != connections)
-  {
-    GNUNET_CONTAINER_multishortmap_destroy (connections);
-    connections = NULL;
-  }
-  if (NULL != ats_ch)
-  {
-    GNUNET_ATS_connectivity_done (ats_ch);
-    ats_ch = NULL;
-  }
-  GCD_shutdown ();
-  GCH_shutdown ();
-  GNUNET_free_non_null (my_private_key);
-  my_private_key = NULL;
-}
-
-
-/**
- * Task run during shutdown.
- *
- * @param cls unused
- */
-static void
-shutdown_task (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Shutting down\n");
-  shutting_down = GNUNET_YES;
-  GCO_shutdown ();
-  if (NULL == clients_head)
-    shutdown_rest ();
-}
-
-
-/**
- * We had a remote connection @a value to port @a port before
- * client @a cls opened port @a port.  Bind them now.
- *
- * @param cls the `struct CadetClient`
- * @param port the port
- * @param value the `struct CadetChannel`
- * @return #GNUNET_YES (iterate over all such channels)
- */
-static int
-bind_loose_channel (void *cls,
-                    const struct GNUNET_HashCode *port,
-                    void *value)
-{
-  struct CadetClient *c = cls;
-  struct CadetChannel *ch = value;
-
-  GCCH_bind (ch,
-             c);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (loose_channels,
-                                                       port,
-                                                       value));
-  return GNUNET_YES;
-}
-
-
-/**
- * Handle port open request.  Creates a mapping from the
- * port to the respective client and checks whether we have
- * loose channels trying to bind to the port.  If so, those
- * are bound.
- *
- * @param cls Identification of the client.
- * @param pmsg The actual message.
- */
-static void
-handle_port_open (void *cls,
-                  const struct GNUNET_CADET_PortMessage *pmsg)
-{
-  struct CadetClient *c = cls;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Open port %s requested by %s\n",
-       GNUNET_h2s (&pmsg->port),
-       GSC_2s (c));
-  if (NULL == c->ports)
-    c->ports = GNUNET_CONTAINER_multihashmap_create (4,
-                                                      GNUNET_NO);
-  if (GNUNET_OK !=
-      GNUNET_CONTAINER_multihashmap_put (c->ports,
-                                         &pmsg->port,
-                                         c,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  (void) GNUNET_CONTAINER_multihashmap_put (open_ports,
-                                            &pmsg->port,
-                                            c,
-                                            GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-  GNUNET_CONTAINER_multihashmap_get_multiple (loose_channels,
-                                              &pmsg->port,
-                                              &bind_loose_channel,
-                                              c);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Handler for port close requests.  Marks this port as closed
- * (unless of course we have another client with the same port
- * open).  Note that existing channels accepted on the port are
- * not affected.
- *
- * @param cls Identification of the client.
- * @param pmsg The actual message.
- */
-static void
-handle_port_close (void *cls,
-                   const struct GNUNET_CADET_PortMessage *pmsg)
-{
-  struct CadetClient *c = cls;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Closing port %s as requested by %s\n",
-       GNUNET_h2s (&pmsg->port),
-       GSC_2s (c));
-  if (GNUNET_YES !=
-      GNUNET_CONTAINER_multihashmap_remove (c->ports,
-                                            &pmsg->port,
-                                            c))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (open_ports,
-                                                       &pmsg->port,
-                                                       c));
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Handler for requests for us creating a new channel to another peer and port.
- *
- * @param cls Identification of the client.
- * @param tcm The actual message.
- */
-static void
-handle_channel_create (void *cls,
-                       const struct GNUNET_CADET_LocalChannelCreateMessage *tcm)
-{
-  struct CadetClient *c = cls;
-  struct CadetChannel *ch;
-
-  if (ntohl (tcm->ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-  {
-    /* Channel ID not in allowed range. */
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  ch = lookup_channel (c,
-                       tcm->ccn);
-  if (NULL != ch)
-  {
-    /* Channel ID already in use. Not allowed. */
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "New channel to %s at port %s requested by %s\n",
-       GNUNET_i2s (&tcm->peer),
-       GNUNET_h2s (&tcm->port),
-       GSC_2s (c));
-
-  /* Create channel */
-  ch = GCCH_channel_local_new (c,
-                               tcm->ccn,
-                               GCP_get (&tcm->peer,
-                                        GNUNET_YES),
-                               &tcm->port,
-                               ntohl (tcm->opt));
-  if (NULL == ch)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_put (c->channels,
-                                                      ntohl (tcm->ccn.channel_of_client),
-                                                      ch,
-                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Handler for requests of destroying an existing channel.
- *
- * @param cls client identification of the client
- * @param msg the actual message
- */
-static void
-handle_channel_destroy (void *cls,
-                        const struct GNUNET_CADET_LocalChannelDestroyMessage *msg)
-{
-  struct CadetClient *c = cls;
-  struct CadetChannel *ch;
-
-  ch = lookup_channel (c,
-                       msg->ccn);
-  if (NULL == ch)
-  {
-    /* Client attempted to destroy unknown channel.
-       Can happen if the other side went down at the same time.*/
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "%s tried to destroy unknown channel %X\n",
-         GSC_2s(c),
-         (uint32_t) ntohl (msg->ccn.channel_of_client));
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "%s is destroying %s\n",
-       GSC_2s(c),
-       GCCH_2s (ch));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
-                                                         ntohl (msg->ccn.channel_of_client),
-                                                         ch));
-  GCCH_channel_local_destroy (ch,
-                              c,
-                              msg->ccn);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Check for client traffic data message is well-formed.
- *
- * @param cls identification of the client
- * @param msg the actual message
- * @return #GNUNET_OK if @a msg is OK, #GNUNET_SYSERR if not
- */
-static int
-check_local_data (void *cls,
-                  const struct GNUNET_CADET_LocalData *msg)
-{
-  size_t payload_size;
-  size_t payload_claimed_size;
-  const char *buf;
-  struct GNUNET_MessageHeader pa;
-
-  /* FIXME: what is the format we shall allow for @a msg?
-     ONE payload item or multiple? Seems current cadet_api
-     at least in theory allows more than one. Next-gen
-     cadet_api will likely no more, so we could then
-     simplify this mess again. */
-  /* Sanity check for message size */
-  payload_size = ntohs (msg->header.size) - sizeof (*msg);
-  buf = (const char *) &msg[1];
-  while (payload_size >= sizeof (struct GNUNET_MessageHeader))
-  {
-    /* need to memcpy() for alignment */
-    GNUNET_memcpy (&pa,
-                   buf,
-                   sizeof (pa));
-    payload_claimed_size = ntohs (pa.size);
-    if ( (payload_size < payload_claimed_size) ||
-         (payload_claimed_size < sizeof (struct GNUNET_MessageHeader)) ||
-         (GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE < payload_claimed_size) )
-    {
-      GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Local data of %u total size had sub-message %u at %u with %u bytes\n",
-           ntohs (msg->header.size),
-           ntohs (pa.type),
-           (unsigned int) (buf - (const char *) &msg[1]),
-           (unsigned int) payload_claimed_size);
-      return GNUNET_SYSERR;
-    }
-    payload_size -= payload_claimed_size;
-    buf += payload_claimed_size;
-  }
-  if (0 != payload_size)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Handler for client payload traffic to be send on a channel to
- * another peer.
- *
- * @param cls identification of the client
- * @param msg the actual message
- */
-static void
-handle_local_data (void *cls,
-                   const struct GNUNET_CADET_LocalData *msg)
-{
-  struct CadetClient *c = cls;
-  struct CadetChannel *ch;
-  size_t payload_size;
-  const char *buf;
-
-  ch = lookup_channel (c,
-                       msg->ccn);
-  if (NULL == ch)
-  {
-    /* Channel does not exist (anymore) */
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Dropping payload for channel %u from client (channel unknown, other endpoint may have disconnected)\n",
-         (unsigned int) ntohl (msg->ccn.channel_of_client));
-    GNUNET_SERVICE_client_continue (c->client);
-    return;
-  }
-  payload_size = ntohs (msg->header.size) - sizeof (*msg);
-  buf = (const char *) &msg[1];
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received %u bytes payload from %s for %s\n",
-       (unsigned int) payload_size,
-       GSC_2s (c),
-       GCCH_2s (ch));
-  if (GNUNET_OK !=
-      GCCH_handle_local_data (ch,
-                              msg->ccn,
-                              buf,
-                              payload_size))
-  {
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Handler for client's ACKs for payload traffic.
- *
- * @param cls identification of the client.
- * @param msg The actual message.
- */
-static void
-handle_local_ack (void *cls,
-                  const struct GNUNET_CADET_LocalAck *msg)
-{
-  struct CadetClient *c = cls;
-  struct CadetChannel *ch;
-
-  ch = lookup_channel (c,
-                       msg->ccn);
-  if (NULL == ch)
-  {
-    /* Channel does not exist! */
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (c->client);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got a local ACK from %s for %s\n",
-       GSC_2s(c),
-       GCCH_2s (ch));
-  GCCH_handle_local_ack (ch,
-                         msg->ccn);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Iterator over all peers to send a monitoring client info about each peer.
- *
- * @param cls Closure ().
- * @param peer Peer ID (tunnel remote peer).
- * @param value Peer info.
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-get_all_peers_iterator (void *cls,
-                        const struct GNUNET_PeerIdentity *peer,
-                        void *value)
-{
-  struct CadetClient *c = cls;
-  struct CadetPeer *p = value;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalInfoPeer *msg;
-
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
-  msg->destination = *peer;
-  msg->paths = htons (GCP_count_paths (p));
-  msg->tunnel = htons (NULL != GCP_get_tunnel (p,
-                                               GNUNET_NO));
-  GNUNET_MQ_send (c->mq,
-                  env);
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's INFO PEERS request.
- *
- * @param cls Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_get_peers (void *cls,
-                  const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c = cls;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *reply;
-
-  GCP_iterate_all (&get_all_peers_iterator,
-                   c);
-  env = GNUNET_MQ_msg (reply,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
-  GNUNET_MQ_send (c->mq,
-                  env);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Iterator over all paths of a peer to build an InfoPeer message.
- * Message contains blocks of peers, first not included.
- *
- * @param cls message queue for transmission
- * @param path Path itself
- * @param off offset of the peer on @a path
- * @return #GNUNET_YES if should keep iterating.
- *         #GNUNET_NO otherwise.
- */
-static int
-path_info_iterator (void *cls,
-                    struct CadetPeerPath *path,
-                    unsigned int off)
-{
-  struct GNUNET_MQ_Handle *mq = cls;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *resp;
-  struct GNUNET_PeerIdentity *id;
-  uint16_t path_size;
-  unsigned int i;
-  unsigned int path_length;
-
-  path_length = GCPP_get_length (path);
-  path_size = sizeof (struct GNUNET_PeerIdentity) * (path_length - 1);
-  if (sizeof (*resp) + path_size > UINT16_MAX)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Path of %u entries is too long for info message\n",
-         path_length);
-    return GNUNET_YES;
-  }
-  env = GNUNET_MQ_msg_extra (resp,
-                             path_size,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
-  id = (struct GNUNET_PeerIdentity *) &resp[1];
-
-  /* Don't copy first peer.  First peer is always the local one.  Last
-   * peer is always the destination (leave as 0, EOL).
-   */
-  for (i = 0; i < off; i++)
-    id[i] = *GCP_get_id (GCPP_get_peer_at_offset (path,
-                                                  i + 1));
-  GNUNET_MQ_send (mq,
-                  env);
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's SHOW_PEER request.
- *
- * @param cls Identification of the client.
- * @param msg The actual message.
- */
-static void
-handle_show_peer (void *cls,
-                  const struct GNUNET_CADET_LocalInfo *msg)
-{
-  struct CadetClient *c = cls;
-  struct CadetPeer *p;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *resp;
-
-  p = GCP_get (&msg->peer,
-               GNUNET_NO);
-  if (NULL != p)
-    GCP_iterate_paths (p,
-                       &path_info_iterator,
-                       c->mq);
-  /* Send message with 0/0 to indicate the end */
-  env = GNUNET_MQ_msg (resp,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER_END);
-  GNUNET_MQ_send (c->mq,
-                  env);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Iterator over all tunnels to send a monitoring client info about each tunnel.
- *
- * @param cls Closure ().
- * @param peer Peer ID (tunnel remote peer).
- * @param value a `struct CadetPeer`
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-get_all_tunnels_iterator (void *cls,
-                          const struct GNUNET_PeerIdentity *peer,
-                          void *value)
-{
-  struct CadetClient *c = cls;
-  struct CadetPeer *p = value;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalInfoTunnel *msg;
-  struct CadetTunnel *t;
-
-  t = GCP_get_tunnel (p,
-                      GNUNET_NO);
-  if (NULL == t)
-    return GNUNET_YES;
-  env = GNUNET_MQ_msg (msg,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
-  msg->destination = *peer;
-  msg->channels = htonl (GCT_count_channels (t));
-  msg->connections = htonl (GCT_count_any_connections (t));
-  msg->cstate = htons (0);
-  msg->estate = htons ((uint16_t) GCT_get_estate (t));
-  GNUNET_MQ_send (c->mq,
-                  env);
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS request.
- *
- * @param cls client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_info_tunnels (void *cls,
-                     const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c = cls;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *reply;
-
-  GCP_iterate_all (&get_all_tunnels_iterator,
-                   c);
-  env = GNUNET_MQ_msg (reply,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
-  GNUNET_MQ_send (c->mq,
-                  env);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Update the message with information about the connection.
- *
- * @param cls a `struct GNUNET_CADET_LocalInfoTunnel` message to update
- * @param ct a connection about which we should store information in @a cls
- */
-static void
-iter_connection (void *cls,
-                 struct CadetTConnection *ct)
-{
-  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
-  struct CadetConnection *cc = ct->cc;
-  struct GNUNET_CADET_ConnectionTunnelIdentifier *h;
-
-  h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
-  h[msg->connections++] = *(GCC_get_id (cc));
-}
-
-
-/**
- * Update the message with information about the channel.
- *
- * @param cls a `struct GNUNET_CADET_LocalInfoTunnel` message to update
- * @param ch a channel about which we should store information in @a cls
- */
-static void
-iter_channel (void *cls,
-              struct CadetChannel *ch)
-{
-  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
-  struct GNUNET_CADET_ConnectionTunnelIdentifier *h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
-  struct GNUNET_CADET_ChannelTunnelNumber *chn
-    = (struct GNUNET_CADET_ChannelTunnelNumber *) &h[msg->connections];
-
-  chn[msg->channels++] = GCCH_get_id (ch);
-}
-
-
-/**
- * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL request.
- *
- * @param cls Identification of the client.
- * @param msg The actual message.
- */
-static void
-handle_info_tunnel (void *cls,
-                    const struct GNUNET_CADET_LocalInfo *msg)
-{
-  struct CadetClient *c = cls;
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalInfoTunnel *resp;
-  struct CadetTunnel *t;
-  struct CadetPeer *p;
-  unsigned int ch_n;
-  unsigned int c_n;
-
-  p = GCP_get (&msg->peer,
-               GNUNET_NO);
-  t = GCP_get_tunnel (p,
-                      GNUNET_NO);
-  if (NULL == t)
-  {
-    /* We don't know the tunnel */
-    struct GNUNET_MQ_Envelope *env;
-    struct GNUNET_CADET_LocalInfoTunnel *warn;
-
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         "Tunnel to %s unknown\n",
-         GNUNET_i2s_full (&msg->peer));
-    env = GNUNET_MQ_msg (warn,
-                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
-    warn->destination = msg->peer;
-    GNUNET_MQ_send (c->mq,
-                    env);
-    GNUNET_SERVICE_client_continue (c->client);
-    return;
-  }
-
-  /* Initialize context */
-  ch_n = GCT_count_channels (t);
-  c_n = GCT_count_any_connections (t);
-  env = GNUNET_MQ_msg_extra (resp,
-                             c_n * sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier) +
-                             ch_n * sizeof (struct GNUNET_CADET_ChannelTunnelNumber),
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
-  resp->destination = msg->peer;
-  /* Do not reorder! #iter_channel needs counters in HBO! */
-  GCT_iterate_connections (t,
-                           &iter_connection,
-                           resp);
-  GCT_iterate_channels (t,
-                        &iter_channel,
-                        resp);
-  resp->connections = htonl (resp->connections);
-  resp->channels = htonl (resp->channels);
-  resp->cstate = htons (0);
-  resp->estate = htons (GCT_get_estate (t));
-  GNUNET_MQ_send (c->mq,
-                  env);
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-/**
- * Iterator over all peers to dump info for each peer.
- *
- * @param cls Closure (unused).
- * @param peer Peer ID (tunnel remote peer).
- * @param value Peer info.
- *
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-show_peer_iterator (void *cls,
-                    const struct GNUNET_PeerIdentity *peer,
-                    void *value)
-{
-  struct CadetPeer *p = value;
-  struct CadetTunnel *t;
-
-  t = GCP_get_tunnel (p,
-                      GNUNET_NO);
-  if (NULL != t)
-    GCT_debug (t,
-               GNUNET_ERROR_TYPE_ERROR);
-  LOG (GNUNET_ERROR_TYPE_ERROR, "\n");
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's INFO_DUMP request.
- *
- * @param cls Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_info_dump (void *cls,
-                  const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c = cls;
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "Received dump info request from client %u\n",
-       c->id);
-
-  LOG (GNUNET_ERROR_TYPE_ERROR,
-       "*************************** DUMP START ***************************\n");
-  for (struct CadetClient *ci = clients_head;
-       NULL != ci;
-       ci = ci->next)
-  {
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         "Client %u (%p), handle: %p, ports: %u, channels: %u\n",
-         ci->id,
-         ci,
-         ci->client,
-         (NULL != c->ports)
-         ? GNUNET_CONTAINER_multihashmap_size (ci->ports)
-         : 0,
-         GNUNET_CONTAINER_multihashmap32_size (ci->channels));
-  }
-  LOG (GNUNET_ERROR_TYPE_ERROR, "***************************\n");
-  GCP_iterate_all (&show_peer_iterator,
-                   NULL);
-
-  LOG (GNUNET_ERROR_TYPE_ERROR,
-       "**************************** DUMP END ****************************\n");
-
-  GNUNET_SERVICE_client_continue (c->client);
-}
-
-
-
-/**
- * Callback called when a client connects to the service.
- *
- * @param cls closure for the service
- * @param client the new client that connected to the service
- * @param mq the message queue used to send messages to the client
- * @return @a c
- */
-static void *
-client_connect_cb (void *cls,
-                   struct GNUNET_SERVICE_Client *client,
-                   struct GNUNET_MQ_Handle *mq)
-{
-  struct CadetClient *c;
-
-  c = GNUNET_new (struct CadetClient);
-  c->client = client;
-  c->mq = mq;
-  c->id = next_client_id++; /* overflow not important: just for debug */
-  c->channels
-    = GNUNET_CONTAINER_multihashmap32_create (32);
-  GNUNET_CONTAINER_DLL_insert (clients_head,
-                               clients_tail,
-                               c);
-  GNUNET_STATISTICS_update (stats,
-                            "# clients",
-                            +1,
-                            GNUNET_NO);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "%s connected\n",
-       GSC_2s (c));
-  return c;
-}
-
-
-/**
- * A channel was destroyed by the other peer. Tell our client.
- *
- * @param c client that lost a channel
- * @param ccn channel identification number for the client
- * @param ch the channel object
- */
-void
-GSC_handle_remote_channel_destroy (struct CadetClient *c,
-                                   struct GNUNET_CADET_ClientChannelNumber ccn,
-                                   struct CadetChannel *ch)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalChannelDestroyMessage *tdm;
-
-  env = GNUNET_MQ_msg (tdm,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
-  tdm->ccn = ccn;
-  GSC_send_to_client (c,
-                      env);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
-                                                         ntohl (ccn.channel_of_client),
-                                                         ch));
-}
-
-
-/**
- * A client that created a loose channel that was not bound to a port
- * disconnected, drop it from the #loose_channels list.
- *
- * @param port the port the channel was trying to bind to
- * @param ch the channel that was lost
- */
-void
-GSC_drop_loose_channel (const struct GNUNET_HashCode *port,
-                        struct CadetChannel *ch)
-{
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (loose_channels,
-                                                       port,
-                                                       ch));
-}
-
-
-/**
- * Iterator for deleting each channel whose client endpoint disconnected.
- *
- * @param cls Closure (client that has disconnected).
- * @param key The local channel id in host byte order
- * @param value The value stored at the key (channel to destroy).
- * @return #GNUNET_OK, keep iterating.
- */
-static int
-channel_destroy_iterator (void *cls,
-                          uint32_t key,
-                          void *value)
-{
-  struct CadetClient *c = cls;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-  struct CadetChannel *ch = value;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying %s, due to %s disconnecting.\n",
-       GCCH_2s (ch),
-       GSC_2s (c));
-  ccn.channel_of_client = htonl (key);
-  GCCH_channel_local_destroy (ch,
-                              c,
-                              ccn);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
-                                                         key,
-                                                         ch));
-  return GNUNET_OK;
-}
-
-
-/**
- * Remove client's ports from the global hashmap on disconnect.
- *
- * @param cls Closure (unused).
- * @param key the port.
- * @param value the `struct CadetClient` to remove
- * @return #GNUNET_OK, keep iterating.
- */
-static int
-client_release_ports (void *cls,
-                      const struct GNUNET_HashCode *key,
-                      void *value)
-{
-  struct CadetClient *c = value;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Closing port %s due to %s disconnect.\n",
-       GNUNET_h2s (key),
-       GSC_2s (c));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (open_ports,
-                                                       key,
-                                                       value));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multihashmap_remove (c->ports,
-                                                       key,
-                                                       value));
-  return GNUNET_OK;
-}
-
-
-/**
- * Callback called when a client disconnected from the service
- *
- * @param cls closure for the service
- * @param client the client that disconnected
- * @param internal_cls should be equal to @a c
- */
-static void
-client_disconnect_cb (void *cls,
-                      struct GNUNET_SERVICE_Client *client,
-                      void *internal_cls)
-{
-  struct CadetClient *c = internal_cls;
-
-  GNUNET_assert (c->client == client);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "%s is disconnecting.\n",
-       GSC_2s (c));
-  if (NULL != c->channels)
-  {
-    GNUNET_CONTAINER_multihashmap32_iterate (c->channels,
-                                             &channel_destroy_iterator,
-                                             c);
-    GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap32_size (c->channels));
-    GNUNET_CONTAINER_multihashmap32_destroy (c->channels);
-  }
-  if (NULL != c->ports)
-  {
-    GNUNET_CONTAINER_multihashmap_iterate (c->ports,
-                                           &client_release_ports,
-                                           c);
-    GNUNET_CONTAINER_multihashmap_destroy (c->ports);
-  }
-  GNUNET_CONTAINER_DLL_remove (clients_head,
-                               clients_tail,
-                               c);
-  GNUNET_STATISTICS_update (stats,
-                            "# clients",
-                            -1,
-                            GNUNET_NO);
-  GNUNET_free (c);
-  if ( (NULL == clients_head) &&
-       (GNUNET_YES == shutting_down) )
-    shutdown_rest ();
-}
-
-
-/**
- * Setup CADET internals.
- *
- * @param cls closure
- * @param server the initialized server
- * @param c configuration to use
- */
-static void
-run (void *cls,
-     const struct GNUNET_CONFIGURATION_Handle *c,
-     struct GNUNET_SERVICE_Handle *service)
-{
-  cfg = c;
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c,
-                                             "CADET",
-                                             "RATCHET_MESSAGES",
-                                             &ratchet_messages))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET",
-                               "RATCHET_MESSAGES",
-                               "needs to be a number");
-    ratchet_messages = 64;
-  }
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c,
-                                           "CADET",
-                                           "RATCHET_TIME",
-                                           &ratchet_time))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET",
-                               "RATCHET_TIME",
-                               "need delay value");
-    ratchet_time = GNUNET_TIME_UNIT_HOURS;
-  }
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c,
-                                           "CADET",
-                                           "REFRESH_CONNECTION_TIME",
-                                           &keepalive_period))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET",
-                               "REFRESH_CONNECTION_TIME",
-                               "need delay value");
-    keepalive_period = GNUNET_TIME_UNIT_MINUTES;
-  }
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c,
-                                             "CADET",
-                                             "DROP_PERCENT",
-                                             &drop_percent))
-  {
-    drop_percent = 0;
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
-    LOG (GNUNET_ERROR_TYPE_WARNING, "Cadet is running with DROP enabled.\n");
-    LOG (GNUNET_ERROR_TYPE_WARNING, "This is NOT a good idea!\n");
-    LOG (GNUNET_ERROR_TYPE_WARNING, "Remove DROP_PERCENT from config file.\n");
-    LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
-  }
-  my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (c);
-  if (NULL == my_private_key)
-  {
-    GNUNET_break (0);
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  GNUNET_CRYPTO_eddsa_key_get_public (my_private_key,
-                                      &my_full_id.public_key);
-  stats = GNUNET_STATISTICS_create ("cadet",
-                                    c);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
-                                 NULL);
-  ats_ch = GNUNET_ATS_connectivity_init (c);
-  /* FIXME: optimize code to allow GNUNET_YES here! */
-  open_ports = GNUNET_CONTAINER_multihashmap_create (16,
-                                                     GNUNET_NO);
-  loose_channels = GNUNET_CONTAINER_multihashmap_create (16,
-                                                         GNUNET_NO);
-  peers = GNUNET_CONTAINER_multipeermap_create (16,
-                                                GNUNET_YES);
-  connections = GNUNET_CONTAINER_multishortmap_create (256,
-                                                       GNUNET_YES);
-  GCH_init (c);
-  GCD_init (c);
-  GCO_init (c);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "CADET started for peer %s\n",
-              GNUNET_i2s (&my_full_id));
-
-}
-
-
-/**
- * Define "main" method using service macro.
- */
-GNUNET_SERVICE_MAIN
-("cadet",
- GNUNET_SERVICE_OPTION_NONE,
- &run,
- &client_connect_cb,
- &client_disconnect_cb,
- NULL,
- GNUNET_MQ_hd_fixed_size (port_open,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN,
-                          struct GNUNET_CADET_PortMessage,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (port_close,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE,
-                          struct GNUNET_CADET_PortMessage,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (channel_create,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE,
-                          struct GNUNET_CADET_LocalChannelCreateMessage,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (channel_destroy,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY,
-                          struct GNUNET_CADET_LocalChannelDestroyMessage,
-                          NULL),
- GNUNET_MQ_hd_var_size (local_data,
-                        GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
-                        struct GNUNET_CADET_LocalData,
-                        NULL),
- GNUNET_MQ_hd_fixed_size (local_ack,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
-                          struct GNUNET_CADET_LocalAck,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (get_peers,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
-                          struct GNUNET_MessageHeader,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (show_peer,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
-                          struct GNUNET_CADET_LocalInfo,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (info_tunnels,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
-                          struct GNUNET_MessageHeader,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (info_tunnel,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
-                          struct GNUNET_CADET_LocalInfo,
-                          NULL),
- GNUNET_MQ_hd_fixed_size (info_dump,
-                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP,
-                          struct GNUNET_MessageHeader,
-                          NULL),
- GNUNET_MQ_handler_end ());
-
-/* end of gnunet-service-cadet-new.c */
diff --git a/src/cadet/gnunet-service-cadet-new_channel.c b/src/cadet/gnunet-service-cadet-new_channel.c
deleted file mode 100644
index 8769601c2..000000000
--- a/src/cadet/gnunet-service-cadet-new_channel.c
+++ /dev/null
@@ -1,2037 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/gnunet-service-cadet-new_channel.c
- * @brief logical links between CADET clients
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * TODO:
- * - Congestion/flow control:
- *   + estimate max bandwidth using bursts and use to for CONGESTION CONTROL!
- *     (and figure out how/where to use this!)
- *   + figure out flow control without ACKs (unreliable traffic!)
- * - revisit handling of 'unbuffered' traffic!
- *   (need to push down through tunnel into connection selection)
- * - revisit handling of 'buffered' traffic: 4 is a rather small buffer; maybe
- *   reserve more bits in 'options' to allow for buffer size control?
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "cadet.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_channel.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
-
-#define LOG(level,...) GNUNET_log_from (level,"cadet-chn",__VA_ARGS__)
-
-/**
- * How long do we initially wait before retransmitting?
- */
-#define CADET_INITIAL_RETRANSMIT_TIME GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 250)
-
-/**
- * How long do we wait before dropping state about incoming
- * connection to closed port?
- */
-#define TIMEOUT_CLOSED_PORT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 30)
-
-/**
- * How long do we wait at least before retransmitting ever?
- */
-#define MIN_RTT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 75)
-
-/**
- * Maximum message ID into the future we accept for out-of-order messages.
- * If the message is more than this into the future, we drop it.  This is
- * important both to detect values that are actually in the past, as well
- * as to limit adversarially triggerable memory consumption.
- *
- * Note that right now we have "max_pending_messages = 4" hard-coded in
- * the logic below, so a value of 4 would suffice here. But we plan to
- * allow larger windows in the future...
- */
-#define MAX_OUT_OF_ORDER_DISTANCE 1024
-
-
-/**
- * All the states a channel can be in.
- */
-enum CadetChannelState
-{
-  /**
-   * Uninitialized status, should never appear in operation.
-   */
-  CADET_CHANNEL_NEW,
-
-  /**
-   * Channel is to a port that is not open, we're waiting for the
-   * port to be opened.
-   */
-  CADET_CHANNEL_LOOSE,
-
-  /**
-   * CHANNEL_OPEN message sent, waiting for CHANNEL_OPEN_ACK.
-   */
-  CADET_CHANNEL_OPEN_SENT,
-
-  /**
-   * Connection confirmed, ready to carry traffic.
-   */
-  CADET_CHANNEL_READY
-};
-
-
-/**
- * Info needed to retry a message in case it gets lost.
- * Note that we DO use this structure also for unreliable
- * messages.
- */
-struct CadetReliableMessage
-{
-  /**
-   * Double linked list, FIFO style
-   */
-  struct CadetReliableMessage *next;
-
-  /**
-   * Double linked list, FIFO style
-   */
-  struct CadetReliableMessage *prev;
-
-  /**
-   * Which channel is this message in?
-   */
-  struct CadetChannel *ch;
-
-  /**
-   * Entry in the tunnels queue for this message, NULL if it has left
-   * the tunnel.  Used to cancel transmission in case we receive an
-   * ACK in time.
-   */
-  struct CadetTunnelQueueEntry *qe;
-
-  /**
-   * Data message we are trying to send.
-   */
-  struct GNUNET_CADET_ChannelAppDataMessage *data_message;
-
-  /**
-   * How soon should we retry if we fail to get an ACK?
-   * Messages in the queue are sorted by this value.
-   */
-  struct GNUNET_TIME_Absolute next_retry;
-
-  /**
-   * How long do we wait for an ACK after transmission?
-   * Use for the back-off calculation.
-   */
-  struct GNUNET_TIME_Relative retry_delay;
-
-  /**
-   * Time when we first successfully transmitted the message
-   * (that is, set @e num_transmissions to 1).
-   */
-  struct GNUNET_TIME_Absolute first_transmission_time;
-
-  /**
-   * Identifier of the connection that this message took when it
-   * was first transmitted.  Only useful if @e num_transmissions is 1.
-   */
-  struct GNUNET_CADET_ConnectionTunnelIdentifier connection_taken;
-
-  /**
-   * How often was this message transmitted?  #GNUNET_SYSERR if there
-   * was an error transmitting the message, #GNUNET_NO if it was not
-   * yet transmitted ever, otherwise the number of (re) transmissions.
-   */
-  int num_transmissions;
-
-};
-
-
-/**
- * List of received out-of-order data messages.
- */
-struct CadetOutOfOrderMessage
-{
-  /**
-   * Double linked list, FIFO style
-   */
-  struct CadetOutOfOrderMessage *next;
-
-  /**
-   * Double linked list, FIFO style
-   */
-  struct CadetOutOfOrderMessage *prev;
-
-  /**
-   * ID of the message (messages up to this point needed
-   * before we give this one to the client).
-   */
-  struct ChannelMessageIdentifier mid;
-
-  /**
-   * The envelope with the payload of the out-of-order message
-   */
-  struct GNUNET_MQ_Envelope *env;
-
-};
-
-
-/**
- * Client endpoint of a `struct CadetChannel`.  A channel may be a
- * loopback channel, in which case it has two of these endpoints.
- * Note that flow control also is required in both directions.
- */
-struct CadetChannelClient
-{
-  /**
-   * Client handle.  Not by itself sufficient to designate
-   * the client endpoint, as the same client handle may
-   * be used for both the owner and the destination, and
-   * we thus also need the channel ID to identify the client.
-   */
-  struct CadetClient *c;
-
-  /**
-   * Head of DLL of messages received out of order or while client was unready.
-   */
-  struct CadetOutOfOrderMessage *head_recv;
-
-  /**
-   * Tail DLL of messages received out of order or while client was unready.
-   */
-  struct CadetOutOfOrderMessage *tail_recv;
-
-  /**
-   * Local tunnel number for this client.
-   * (if owner >= #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI,
-   *  otherwise < #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-   */
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  /**
-   * Number of entries currently in @a head_recv DLL.
-   */
-  unsigned int num_recv;
-
-  /**
-   * Can we send data to the client?
-   */
-  int client_ready;
-
-};
-
-
-/**
- * Struct containing all information regarding a channel to a remote client.
- */
-struct CadetChannel
-{
-  /**
-   * Tunnel this channel is in.
-   */
-  struct CadetTunnel *t;
-
-  /**
-   * Client owner of the tunnel, if any.
-   * (Used if this channel represends the initiating end of the tunnel.)
-   */
-  struct CadetChannelClient *owner;
-
-  /**
-   * Client destination of the tunnel, if any.
-   * (Used if this channel represents the listening end of the tunnel.)
-   */
-  struct CadetChannelClient *dest;
-
-  /**
-   * Last entry in the tunnel's queue relating to control messages
-   * (#GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN or
-   * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK).  Used to cancel
-   * transmission in case we receive updated information.
-   */
-  struct CadetTunnelQueueEntry *last_control_qe;
-
-  /**
-   * Head of DLL of messages sent and not yet ACK'd.
-   */
-  struct CadetReliableMessage *head_sent;
-
-  /**
-   * Tail of DLL of messages sent and not yet ACK'd.
-   */
-  struct CadetReliableMessage *tail_sent;
-
-  /**
-   * Task to resend/poll in case no ACK is received.
-   */
-  struct GNUNET_SCHEDULER_Task *retry_control_task;
-
-  /**
-   * Task to resend/poll in case no ACK is received.
-   */
-  struct GNUNET_SCHEDULER_Task *retry_data_task;
-
-  /**
-   * Last time the channel was used
-   */
-  struct GNUNET_TIME_Absolute timestamp;
-
-  /**
-   * Destination port of the channel.
-   */
-  struct GNUNET_HashCode port;
-
-  /**
-   * Counter for exponential backoff.
-   */
-  struct GNUNET_TIME_Relative retry_time;
-
-  /**
-   * Bitfield of already-received messages past @e mid_recv.
-   */
-  uint64_t mid_futures;
-
-  /**
-   * Next MID expected for incoming traffic.
-   */
-  struct ChannelMessageIdentifier mid_recv;
-
-  /**
-   * Next MID to use for outgoing traffic.
-   */
-  struct ChannelMessageIdentifier mid_send;
-
-  /**
-   * Total (reliable) messages pending ACK for this channel.
-   */
-  unsigned int pending_messages;
-
-  /**
-   * Maximum (reliable) messages pending ACK for this channel
-   * before we throttle the client.
-   */
-  unsigned int max_pending_messages;
-
-  /**
-   * Number identifying this channel in its tunnel.
-   */
-  struct GNUNET_CADET_ChannelTunnelNumber ctn;
-
-  /**
-   * Channel state.
-   */
-  enum CadetChannelState state;
-
-  /**
-   * Count how many ACKs we skipped, used to prevent long
-   * sequences of ACK skipping.
-   */
-  unsigned int skip_ack_series;
-
-  /**
-   * Is the tunnel bufferless (minimum latency)?
-   */
-  int nobuffer;
-
-  /**
-   * Is the tunnel reliable?
-   */
-  int reliable;
-
-  /**
-   * Is the tunnel out-of-order?
-   */
-  int out_of_order;
-
-  /**
-   * Is this channel a loopback channel, where the destination is us again?
-   */
-  int is_loopback;
-
-  /**
-   * Flag to signal the destruction of the channel.  If this is set to
-   * #GNUNET_YES the channel will be destroyed once the queue is
-   * empty.
-   */
-  int destroy;
-
-};
-
-
-/**
- * Get the static string for identification of the channel.
- *
- * @param ch Channel.
- *
- * @return Static string with the channel IDs.
- */
-const char *
-GCCH_2s (const struct CadetChannel *ch)
-{
-  static char buf[128];
-
-  GNUNET_snprintf (buf,
-                   sizeof (buf),
-                   "Channel %s:%s ctn:%X(%X/%X)",
-                   (GNUNET_YES == ch->is_loopback)
-                   ? "loopback"
-                   : GNUNET_i2s (GCP_get_id (GCT_get_destination (ch->t))),
-                   GNUNET_h2s (&ch->port),
-                   ch->ctn,
-                   (NULL == ch->owner) ? 0 : ntohl (ch->owner->ccn.channel_of_client),
-                   (NULL == ch->dest) ? 0 : ntohl (ch->dest->ccn.channel_of_client));
-  return buf;
-}
-
-
-/**
- * Get the channel's public ID.
- *
- * @param ch Channel.
- *
- * @return ID used to identify the channel with the remote peer.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCCH_get_id (const struct CadetChannel *ch)
-{
-  return ch->ctn;
-}
-
-
-/**
- * Release memory associated with @a ccc
- *
- * @param ccc data structure to clean up
- */
-static void
-free_channel_client (struct CadetChannelClient *ccc)
-{
-  struct CadetOutOfOrderMessage *com;
-
-  while (NULL != (com = ccc->head_recv))
-  {
-    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
-                                 ccc->tail_recv,
-                                 com);
-    ccc->num_recv--;
-    GNUNET_MQ_discard (com->env);
-    GNUNET_free (com);
-  }
-  GNUNET_free (ccc);
-}
-
-
-/**
- * Destroy the given channel.
- *
- * @param ch channel to destroy
- */
-static void
-channel_destroy (struct CadetChannel *ch)
-{
-  struct CadetReliableMessage *crm;
-
-  while (NULL != (crm = ch->head_sent))
-  {
-    GNUNET_assert (ch == crm->ch);
-    if (NULL != crm->qe)
-    {
-      GCT_send_cancel (crm->qe);
-      crm->qe = NULL;
-    }
-    GNUNET_CONTAINER_DLL_remove (ch->head_sent,
-                                 ch->tail_sent,
-                                 crm);
-    GNUNET_free (crm->data_message);
-    GNUNET_free (crm);
-  }
-  if (NULL != ch->owner)
-  {
-    free_channel_client (ch->owner);
-    ch->owner = NULL;
-  }
-  if (NULL != ch->dest)
-  {
-    free_channel_client (ch->dest);
-    ch->dest = NULL;
-  }
-  if (NULL != ch->last_control_qe)
-  {
-    GCT_send_cancel (ch->last_control_qe);
-    ch->last_control_qe = NULL;
-  }
-  if (NULL != ch->retry_data_task)
-  {
-    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
-    ch->retry_data_task = NULL;
-  }
-  if (NULL != ch->retry_control_task)
-  {
-    GNUNET_SCHEDULER_cancel (ch->retry_control_task);
-    ch->retry_control_task = NULL;
-  }
-  if (GNUNET_NO == ch->is_loopback)
-  {
-    GCT_remove_channel (ch->t,
-                        ch,
-                        ch->ctn);
-    ch->t = NULL;
-  }
-  GNUNET_free (ch);
-}
-
-
-/**
- * Send a channel create message.
- *
- * @param cls Channel for which to send.
- */
-static void
-send_channel_open (void *cls);
-
-
-/**
- * Function called once the tunnel confirms that we sent the
- * create message.  Delays for a bit until we retry.
- *
- * @param cls our `struct CadetChannel`.
- * @param cid identifier of the connection within the tunnel, NULL
- *            if transmission failed
- */
-static void
-channel_open_sent_cb (void *cls,
-                      const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
-{
-  struct CadetChannel *ch = cls;
-
-  GNUNET_assert (NULL != ch->last_control_qe);
-  ch->last_control_qe = NULL;
-  ch->retry_time = GNUNET_TIME_STD_BACKOFF (ch->retry_time);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sent CADET_CHANNEL_OPEN on %s, retrying in %s\n",
-       GCCH_2s (ch),
-       GNUNET_STRINGS_relative_time_to_string (ch->retry_time,
-                                               GNUNET_YES));
-  ch->retry_control_task
-    = GNUNET_SCHEDULER_add_delayed (ch->retry_time,
-                                    &send_channel_open,
-                                    ch);
-}
-
-
-/**
- * Send a channel open message.
- *
- * @param cls Channel for which to send.
- */
-static void
-send_channel_open (void *cls)
-{
-  struct CadetChannel *ch = cls;
-  struct GNUNET_CADET_ChannelOpenMessage msgcc;
-  uint32_t options;
-
-  ch->retry_control_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending CHANNEL_OPEN message for %s\n",
-       GCCH_2s (ch));
-  options = 0;
-  if (ch->nobuffer)
-    options |= GNUNET_CADET_OPTION_NOBUFFER;
-  if (ch->reliable)
-    options |= GNUNET_CADET_OPTION_RELIABLE;
-  if (ch->out_of_order)
-    options |= GNUNET_CADET_OPTION_OUT_OF_ORDER;
-  msgcc.header.size = htons (sizeof (msgcc));
-  msgcc.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN);
-  msgcc.opt = htonl (options);
-  msgcc.port = ch->port;
-  msgcc.ctn = ch->ctn;
-  ch->state = CADET_CHANNEL_OPEN_SENT;
-  if (NULL != ch->last_control_qe)
-    GCT_send_cancel (ch->last_control_qe);
-  ch->last_control_qe = GCT_send (ch->t,
-                                  &msgcc.header,
-                                  &channel_open_sent_cb,
-                                  ch);
-  GNUNET_assert (NULL == ch->retry_control_task);
-}
-
-
-/**
- * Function called once and only once after a channel was bound
- * to its tunnel via #GCT_add_channel() is ready for transmission.
- * Note that this is only the case for channels that this peer
- * initiates, as for incoming channels we assume that they are
- * ready for transmission immediately upon receiving the open
- * message.  Used to bootstrap the #GCT_send() process.
- *
- * @param ch the channel for which the tunnel is now ready
- */
-void
-GCCH_tunnel_up (struct CadetChannel *ch)
-{
-  GNUNET_assert (NULL == ch->retry_control_task);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Tunnel up, sending CHANNEL_OPEN on %s now\n",
-       GCCH_2s (ch));
-  ch->retry_control_task
-    = GNUNET_SCHEDULER_add_now (&send_channel_open,
-                                ch);
-}
-
-
-/**
- * Create a new channel.
- *
- * @param owner local client owning the channel
- * @param ccn local number of this channel at the @a owner
- * @param destination peer to which we should build the channel
- * @param port desired port at @a destination
- * @param options options for the channel
- * @return handle to the new channel
- */
-struct CadetChannel *
-GCCH_channel_local_new (struct CadetClient *owner,
-                        struct GNUNET_CADET_ClientChannelNumber ccn,
-                        struct CadetPeer *destination,
-                        const struct GNUNET_HashCode *port,
-                        uint32_t options)
-{
-  struct CadetChannel *ch;
-  struct CadetChannelClient *ccco;
-
-  ccco = GNUNET_new (struct CadetChannelClient);
-  ccco->c = owner;
-  ccco->ccn = ccn;
-  ccco->client_ready = GNUNET_YES;
-
-  ch = GNUNET_new (struct CadetChannel);
-  ch->mid_recv.mid = htonl (1); /* The OPEN_ACK counts as message 0! */
-  ch->nobuffer = (0 != (options & GNUNET_CADET_OPTION_NOBUFFER));
-  ch->reliable = (0 != (options & GNUNET_CADET_OPTION_RELIABLE));
-  ch->out_of_order = (0 != (options & GNUNET_CADET_OPTION_OUT_OF_ORDER));
-  ch->max_pending_messages = (ch->nobuffer) ? 1 : 4; /* FIXME: 4!? Do not hardcode! */
-  ch->owner = ccco;
-  ch->port = *port;
-  if (0 == memcmp (&my_full_id,
-                   GCP_get_id (destination),
-                   sizeof (struct GNUNET_PeerIdentity)))
-  {
-    struct CadetClient *c;
-
-    ch->is_loopback = GNUNET_YES;
-    c = GNUNET_CONTAINER_multihashmap_get (open_ports,
-                                           port);
-    if (NULL == c)
-    {
-      /* port closed, wait for it to possibly open */
-      ch->state = CADET_CHANNEL_LOOSE;
-      (void) GNUNET_CONTAINER_multihashmap_put (loose_channels,
-                                                port,
-                                                ch,
-                                                GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Created loose incoming loopback channel to port %s\n",
-           GNUNET_h2s (&ch->port));
-    }
-    else
-    {
-      GCCH_bind (ch,
-                 c);
-    }
-  }
-  else
-  {
-    ch->t = GCP_get_tunnel (destination,
-                            GNUNET_YES);
-    ch->retry_time = CADET_INITIAL_RETRANSMIT_TIME;
-    ch->ctn = GCT_add_channel (ch->t,
-                               ch);
-  }
-  GNUNET_STATISTICS_update (stats,
-                            "# channels",
-                            1,
-                            GNUNET_NO);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Created channel to port %s at peer %s for %s using %s\n",
-       GNUNET_h2s (port),
-       GCP_2s (destination),
-       GSC_2s (owner),
-       (GNUNET_YES == ch->is_loopback) ? "loopback" : GCT_2s (ch->t));
-  return ch;
-}
-
-
-/**
- * We had an incoming channel to a port that is closed.
- * It has not been opened for a while, drop it.
- *
- * @param cls the channel to drop
- */
-static void
-timeout_closed_cb (void *cls)
-{
-  struct CadetChannel *ch = cls;
-
-  ch->retry_control_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Closing incoming channel to port %s from peer %s due to timeout\n",
-       GNUNET_h2s (&ch->port),
-       GCP_2s (GCT_get_destination (ch->t)));
-  channel_destroy (ch);
-}
-
-
-/**
- * Create a new channel based on a request coming in over the network.
- *
- * @param t tunnel to the remote peer
- * @param ctn identifier of this channel in the tunnel
- * @param port desired local port
- * @param options options for the channel
- * @return handle to the new channel
- */
-struct CadetChannel *
-GCCH_channel_incoming_new (struct CadetTunnel *t,
-                           struct GNUNET_CADET_ChannelTunnelNumber ctn,
-                           const struct GNUNET_HashCode *port,
-                           uint32_t options)
-{
-  struct CadetChannel *ch;
-  struct CadetClient *c;
-
-  ch = GNUNET_new (struct CadetChannel);
-  ch->port = *port;
-  ch->t = t;
-  ch->ctn = ctn;
-  ch->retry_time = CADET_INITIAL_RETRANSMIT_TIME;
-  ch->nobuffer = (0 != (options & GNUNET_CADET_OPTION_NOBUFFER));
-  ch->reliable = (0 != (options & GNUNET_CADET_OPTION_RELIABLE));
-  ch->out_of_order = (0 != (options & GNUNET_CADET_OPTION_OUT_OF_ORDER));
-  ch->max_pending_messages = (ch->nobuffer) ? 1 : 4; /* FIXME: 4!? Do not hardcode! */
-  GNUNET_STATISTICS_update (stats,
-                            "# channels",
-                            1,
-                            GNUNET_NO);
-
-  c = GNUNET_CONTAINER_multihashmap_get (open_ports,
-                                         port);
-  if (NULL == c)
-  {
-    /* port closed, wait for it to possibly open */
-    ch->state = CADET_CHANNEL_LOOSE;
-    (void) GNUNET_CONTAINER_multihashmap_put (loose_channels,
-                                              port,
-                                              ch,
-                                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-    GNUNET_assert (NULL == ch->retry_control_task);
-    ch->retry_control_task
-      = GNUNET_SCHEDULER_add_delayed (TIMEOUT_CLOSED_PORT,
-                                      &timeout_closed_cb,
-                                      ch);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Created loose incoming channel to port %s from peer %s\n",
-         GNUNET_h2s (&ch->port),
-         GCP_2s (GCT_get_destination (ch->t)));
-  }
-  else
-  {
-    GCCH_bind (ch,
-               c);
-  }
-  GNUNET_STATISTICS_update (stats,
-                            "# channels",
-                            1,
-                            GNUNET_NO);
-  return ch;
-}
-
-
-/**
- * Function called once the tunnel confirms that we sent the
- * ACK message.  Just remembers it was sent, we do not expect
- * ACKs for ACKs ;-).
- *
- * @param cls our `struct CadetChannel`.
- * @param cid identifier of the connection within the tunnel, NULL
- *            if transmission failed
- */
-static void
-send_ack_cb (void *cls,
-             const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
-{
-  struct CadetChannel *ch = cls;
-
-  GNUNET_assert (NULL != ch->last_control_qe);
-  ch->last_control_qe = NULL;
-}
-
-
-/**
- * Compute and send the current #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK to the other peer.
- *
- * @param ch channel to send the #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK for
- */
-static void
-send_channel_data_ack (struct CadetChannel *ch)
-{
-  struct GNUNET_CADET_ChannelDataAckMessage msg;
-
-  if (GNUNET_NO == ch->reliable)
-    return; /* no ACKs */
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK);
-  msg.header.size = htons (sizeof (msg));
-  msg.ctn = ch->ctn;
-  msg.mid.mid = htonl (ntohl (ch->mid_recv.mid));
-  msg.futures = GNUNET_htonll (ch->mid_futures);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending DATA_ACK %u:%llX via %s\n",
-       (unsigned int) ntohl (msg.mid.mid),
-       (unsigned long long) ch->mid_futures,
-       GCCH_2s (ch));
-  if (NULL != ch->last_control_qe)
-    GCT_send_cancel (ch->last_control_qe);
-  ch->last_control_qe = GCT_send (ch->t,
-                                  &msg.header,
-                                  &send_ack_cb,
-                                  ch);
-}
-
-
-/**
- * Send our initial #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK to the client confirming that the
- * connection is up.
- *
- * @param cls the `struct CadetChannel`
- */
-static void
-send_open_ack (void *cls)
-{
-  struct CadetChannel *ch = cls;
-  struct GNUNET_CADET_ChannelManageMessage msg;
-
-  ch->retry_control_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending CHANNEL_OPEN_ACK on %s\n",
-       GCCH_2s (ch));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK);
-  msg.header.size = htons (sizeof (msg));
-  msg.reserved = htonl (0);
-  msg.ctn = ch->ctn;
-  if (NULL != ch->last_control_qe)
-    GCT_send_cancel (ch->last_control_qe);
-  ch->last_control_qe = GCT_send (ch->t,
-                                  &msg.header,
-                                  &send_ack_cb,
-                                  ch);
-}
-
-
-/**
- * We got a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN message again for
- * this channel.  If the binding was successful, (re)transmit the
- * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
- *
- * @param ch channel that got the duplicate open
- * @param cti identifier of the connection that delivered the message
- */
-void
-GCCH_handle_duplicate_open (struct CadetChannel *ch,
-                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti)
-{
-  if (NULL == ch->dest)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Ignoring duplicate CHANNEL_OPEN on %s: port is closed\n",
-         GCCH_2s (ch));
-    return;
-  }
-  if (NULL != ch->retry_control_task)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Ignoring duplicate CHANNEL_OPEN on %s: control message is pending\n",
-         GCCH_2s (ch));
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Retransmitting CHANNEL_OPEN_ACK on %s\n",
-       GCCH_2s (ch));
-  ch->retry_control_task
-    = GNUNET_SCHEDULER_add_now (&send_open_ack,
-                                ch);
-}
-
-
-/**
- * Send a #GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK to the client to solicit more messages.
- *
- * @param ch channel the ack is for
- * @param to_owner #GNUNET_YES to send to owner,
- *                 #GNUNET_NO to send to dest
- */
-static void
-send_ack_to_client (struct CadetChannel *ch,
-                    int to_owner)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalAck *ack;
-  struct CadetChannelClient *ccc;
-
-  ccc = (GNUNET_YES == to_owner) ? ch->owner : ch->dest;
-  if (NULL == ccc)
-  {
-    /* This can happen if we are just getting ACKs after
-       our local client already disconnected. */
-    GNUNET_assert (GNUNET_YES == ch->destroy);
-    return;
-  }
-  env = GNUNET_MQ_msg (ack,
-                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
-  ack->ccn = ccc->ccn;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending CADET_LOCAL_ACK to %s (%s) at ccn %X (%u/%u pending)\n",
-       GSC_2s (ccc->c),
-       (GNUNET_YES == to_owner) ? "owner" : "dest",
-       ntohl (ack->ccn.channel_of_client),
-       ch->pending_messages,
-       ch->max_pending_messages);
-  GSC_send_to_client (ccc->c,
-                      env);
-}
-
-
-/**
- * A client is bound to the port that we have a channel
- * open to.  Send the acknowledgement for the connection
- * request and establish the link with the client.
- *
- * @param ch open incoming channel
- * @param c client listening on the respective port
- */
-void
-GCCH_bind (struct CadetChannel *ch,
-           struct CadetClient *c)
-{
-  uint32_t options;
-  struct CadetChannelClient *cccd;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Binding %s from %s to port %s of %s\n",
-       GCCH_2s (ch),
-       GCT_2s (ch->t),
-       GNUNET_h2s (&ch->port),
-       GSC_2s (c));
-  if (NULL != ch->retry_control_task)
-  {
-    /* there might be a timeout task here */
-    GNUNET_SCHEDULER_cancel (ch->retry_control_task);
-    ch->retry_control_task = NULL;
-  }
-  options = 0;
-  if (ch->nobuffer)
-    options |= GNUNET_CADET_OPTION_NOBUFFER;
-  if (ch->reliable)
-    options |= GNUNET_CADET_OPTION_RELIABLE;
-  if (ch->out_of_order)
-    options |= GNUNET_CADET_OPTION_OUT_OF_ORDER;
-  cccd = GNUNET_new (struct CadetChannelClient);
-  GNUNET_assert (NULL == ch->dest);
-  ch->dest = cccd;
-  cccd->c = c;
-  cccd->client_ready = GNUNET_YES;
-  cccd->ccn = GSC_bind (c,
-                        ch,
-                        (GNUNET_YES == ch->is_loopback)
-                        ? GCP_get (&my_full_id,
-                                   GNUNET_YES)
-                        : GCT_get_destination (ch->t),
-                        &ch->port,
-                        options);
-  GNUNET_assert (ntohl (cccd->ccn.channel_of_client) <
-                 GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-  ch->mid_recv.mid = htonl (1); /* The OPEN counts as message 0! */
-  if (GNUNET_YES == ch->is_loopback)
-  {
-    ch->state = CADET_CHANNEL_OPEN_SENT;
-    GCCH_handle_channel_open_ack (ch,
-                                  NULL);
-  }
-  else
-  {
-    /* notify other peer that we accepted the connection */
-    ch->state = CADET_CHANNEL_READY;
-    ch->retry_control_task
-      = GNUNET_SCHEDULER_add_now (&send_open_ack,
-                                  ch);
-  }
-  /* give client it's initial supply of ACKs */
-  GNUNET_assert (ntohl (cccd->ccn.channel_of_client) <
-                 GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-  for (unsigned int i=0;i<ch->max_pending_messages;i++)
-    send_ack_to_client (ch,
-                        GNUNET_NO);
-}
-
-
-/**
- * One of our clients has disconnected, tell the other one that we
- * are finished. Done asynchronously to avoid concurrent modification
- * issues if this is the same client.
- *
- * @param cls the `struct CadetChannel` where one of the ends is now dead
- */
-static void
-signal_remote_destroy_cb (void *cls)
-{
-  struct CadetChannel *ch = cls;
-  struct CadetChannelClient *ccc;
-
-  /* Find which end is left... */
-  ch->retry_control_task = NULL;
-  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
-  GSC_handle_remote_channel_destroy (ccc->c,
-                                     ccc->ccn,
-                                     ch);
-  channel_destroy (ch);
-}
-
-
-/**
- * Destroy locally created channel.  Called by the local client, so no
- * need to tell the client.
- *
- * @param ch channel to destroy
- * @param c client that caused the destruction
- * @param ccn client number of the client @a c
- */
-void
-GCCH_channel_local_destroy (struct CadetChannel *ch,
-                            struct CadetClient *c,
-                            struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "%s asks for destruction of %s\n",
-       GSC_2s (c),
-       GCCH_2s (ch));
-  GNUNET_assert (NULL != c);
-  if ( (NULL != ch->owner) &&
-       (c == ch->owner->c) &&
-       (ccn.channel_of_client == ch->owner->ccn.channel_of_client) )
-  {
-    free_channel_client (ch->owner);
-    ch->owner = NULL;
-  }
-  else if ( (NULL != ch->dest) &&
-            (c == ch->dest->c) &&
-            (ccn.channel_of_client == ch->dest->ccn.channel_of_client) )
-  {
-    free_channel_client (ch->dest);
-    ch->dest = NULL;
-  }
-  else
-  {
-    GNUNET_assert (0);
-  }
-
-  if (GNUNET_YES == ch->destroy)
-  {
-    /* other end already destroyed, with the local client gone, no need
-       to finish transmissions, just destroy immediately. */
-    channel_destroy (ch);
-    return;
-  }
-  if ( (NULL != ch->head_sent) &&
-       ( (NULL != ch->owner) ||
-         (NULL != ch->dest) ) )
-  {
-    /* Wait for other end to destroy us as well,
-       and otherwise allow send queue to be transmitted first */
-    ch->destroy = GNUNET_YES;
-    return;
-  }
-  if ( (GNUNET_YES == ch->is_loopback) &&
-       ( (NULL != ch->owner) ||
-         (NULL != ch->dest) ) )
-  {
-    if (NULL != ch->retry_control_task)
-      GNUNET_SCHEDULER_cancel (ch->retry_control_task);
-    ch->retry_control_task
-      = GNUNET_SCHEDULER_add_now (&signal_remote_destroy_cb,
-                                  ch);
-    return;
-  }
-  if (GNUNET_NO == ch->is_loopback)
-  {
-    /* If the we ever sent the CHANNEL_CREATE, we need to send a destroy message. */
-    switch (ch->state)
-    {
-    case CADET_CHANNEL_NEW:
-      /* We gave up on a channel that we created as a client to a remote
-         target, but that never went anywhere. Nothing to do here. */
-      break;
-    case CADET_CHANNEL_LOOSE:
-      GSC_drop_loose_channel (&ch->port,
-                              ch);
-      break;
-    default:
-      GCT_send_channel_destroy (ch->t,
-                                ch->ctn);
-    }
-  }
-  /* Nothing left to do, just finish destruction */
-  channel_destroy (ch);
-}
-
-
-/**
- * We got an acknowledgement for the creation of the channel
- * (the port is open on the other side). Begin transmissions.
- *
- * @param ch channel to destroy
- * @param cti identifier of the connection that delivered the message
- */
-void
-GCCH_handle_channel_open_ack (struct CadetChannel *ch,
-                              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti)
-{
-  switch (ch->state)
-  {
-  case CADET_CHANNEL_NEW:
-    /* this should be impossible */
-    GNUNET_break (0);
-    break;
-  case CADET_CHANNEL_LOOSE:
-    /* This makes no sense. */
-    GNUNET_break_op (0);
-    break;
-  case CADET_CHANNEL_OPEN_SENT:
-    if (NULL == ch->owner)
-    {
-      /* We're not the owner, wrong direction! */
-      GNUNET_break_op (0);
-      return;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Received CHANNEL_OPEN_ACK for waiting %s, entering READY state\n",
-         GCCH_2s (ch));
-    if (NULL != ch->retry_control_task) /* can be NULL if ch->is_loopback */
-    {
-      GNUNET_SCHEDULER_cancel (ch->retry_control_task);
-      ch->retry_control_task = NULL;
-    }
-    ch->state = CADET_CHANNEL_READY;
-    /* On first connect, send client as many ACKs as we allow messages
-       to be buffered! */
-    for (unsigned int i=0;i<ch->max_pending_messages;i++)
-      send_ack_to_client (ch,
-                          GNUNET_YES);
-    break;
-  case CADET_CHANNEL_READY:
-    /* duplicate ACK, maybe we retried the CREATE. Ignore. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Received duplicate channel OPEN_ACK for %s\n",
-         GCCH_2s (ch));
-    GNUNET_STATISTICS_update (stats,
-                              "# duplicate CREATE_ACKs",
-                              1,
-                              GNUNET_NO);
-    break;
-  }
-}
-
-
-/**
- * Test if element @a e1 comes before element @a e2.
- *
- * @param cls closure, to a flag where we indicate duplicate packets
- * @param m1 a message of to sort
- * @param m2 another message to sort
- * @return #GNUNET_YES if @e1 < @e2, otherwise #GNUNET_NO
- */
-static int
-is_before (void *cls,
-           struct CadetOutOfOrderMessage *m1,
-           struct CadetOutOfOrderMessage *m2)
-{
-  int *duplicate = cls;
-  uint32_t v1 = ntohl (m1->mid.mid);
-  uint32_t v2 = ntohl (m2->mid.mid);
-  uint32_t delta;
-
-  delta = v2 - v1;
-  if (0 == delta)
-    *duplicate = GNUNET_YES;
-  if (delta > (uint32_t) INT_MAX)
-  {
-    /* in overflow range, we can safely assume we wrapped around */
-    return GNUNET_NO;
-  }
-  else
-  {
-    /* result is small, thus v2 > v1, thus m1 < m2 */
-    return GNUNET_YES;
-  }
-}
-
-
-/**
- * We got payload data for a channel.  Pass it on to the client
- * and send an ACK to the other end (once flow control allows it!)
- *
- * @param ch channel that got data
- * @param cti identifier of the connection that delivered the message
- * @param msg message that was received
- */
-void
-GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
-                                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
-                                    const struct GNUNET_CADET_ChannelAppDataMessage *msg)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_CADET_LocalData *ld;
-  struct CadetChannelClient *ccc;
-  size_t payload_size;
-  struct CadetOutOfOrderMessage *com;
-  int duplicate;
-  uint32_t mid_min;
-  uint32_t mid_max;
-  uint32_t mid_msg;
-  uint32_t delta;
-
-  GNUNET_assert (GNUNET_NO == ch->is_loopback);
-  if ( (GNUNET_YES == ch->destroy) &&
-       (NULL == ch->owner) &&
-       (NULL == ch->dest) )
-  {
-    /* This client is gone, but we still have messages to send to
-       the other end (which is why @a ch is not yet dead).  However,
-       we cannot pass messages to our client anymore. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Dropping incoming payload on %s as this end is already closed\n",
-         GCCH_2s (ch));
-    /* send back DESTROY notification to stop further retransmissions! */
-    GCT_send_channel_destroy (ch->t,
-                              ch->ctn);
-    return;
-  }
-  payload_size = ntohs (msg->header.size) - sizeof (*msg);
-  env = GNUNET_MQ_msg_extra (ld,
-                             payload_size,
-                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
-  ld->ccn = (NULL == ch->dest) ? ch->owner->ccn : ch->dest->ccn;
-  GNUNET_memcpy (&ld[1],
-                 &msg[1],
-                 payload_size);
-  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
-  if ( (GNUNET_YES == ccc->client_ready) &&
-       ( (GNUNET_YES == ch->out_of_order) ||
-         (msg->mid.mid == ch->mid_recv.mid) ) )
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Giving %u bytes of payload with MID %u from %s to client %s\n",
-         (unsigned int) payload_size,
-         ntohl (msg->mid.mid),
-         GCCH_2s (ch),
-         GSC_2s (ccc->c));
-    ccc->client_ready = GNUNET_NO;
-    GSC_send_to_client (ccc->c,
-                        env);
-    ch->mid_recv.mid = htonl (1 + ntohl (ch->mid_recv.mid));
-    ch->mid_futures >>= 1;
-    send_channel_data_ack (ch);
-    return;
-  }
-
-  if (GNUNET_YES == ch->reliable)
-  {
-    /* check if message ought to be dropped because it is ancient/too distant/duplicate */
-    mid_min = ntohl (ch->mid_recv.mid);
-    mid_max = mid_min + ch->max_pending_messages;
-    mid_msg = ntohl (msg->mid.mid);
-    if ( ( (uint32_t) (mid_msg - mid_min) > ch->max_pending_messages) ||
-         ( (uint32_t) (mid_max - mid_msg) > ch->max_pending_messages) )
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "%s at %u drops ancient or far-future message %u\n",
-           GCCH_2s (ch),
-           (unsigned int) mid_min,
-           ntohl (msg->mid.mid));
-
-      GNUNET_STATISTICS_update (stats,
-                                "# duplicate DATA (ancient or future)",
-                                1,
-                                GNUNET_NO);
-      GNUNET_MQ_discard (env);
-      send_channel_data_ack (ch);
-      return;
-    }
-    /* mark bit for future ACKs */
-    delta = mid_msg - mid_min - 1; /* overflow/underflow are OK here */
-    if (delta < 64)
-    {
-      if (0 != (ch->mid_futures & (1LLU << delta)))
-      {
-        /* Duplicate within the queue, drop also */
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "Duplicate payload of %u bytes on %s (mid %u) dropped\n",
-             (unsigned int) payload_size,
-             GCCH_2s (ch),
-             ntohl (msg->mid.mid));
-        GNUNET_STATISTICS_update (stats,
-                                  "# duplicate DATA",
-                                  1,
-                                  GNUNET_NO);
-        GNUNET_MQ_discard (env);
-        send_channel_data_ack (ch);
-        return;
-      }
-      ch->mid_futures |= (1LLU << delta);
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Marked bit %llX for mid %u (base: %u); now: %llX\n",
-           (1LLU << delta),
-           mid_msg,
-           mid_min,
-           ch->mid_futures);
-    }
-  }
-  else /* ! ch->reliable */
-  {
-    /* Channel is unreliable, so we do not ACK. But we also cannot
-       allow buffering everything, so check if we have space... */
-    if (ccc->num_recv >= ch->max_pending_messages)
-    {
-      struct CadetOutOfOrderMessage *drop;
-
-      /* Yep, need to drop. Drop the oldest message in
-         the buffer. */
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Queue full due slow client on %s, dropping oldest message\n",
-           GCCH_2s (ch));
-      GNUNET_STATISTICS_update (stats,
-                                "# messages dropped due to slow client",
-                                1,
-                                GNUNET_NO);
-      drop = ccc->head_recv;
-      GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
-                                   ccc->tail_recv,
-                                   drop);
-      ccc->num_recv--;
-      GNUNET_MQ_discard (drop->env);
-      GNUNET_free (drop);
-    }
-  }
-
-  /* Insert message into sorted out-of-order queue */
-  com = GNUNET_new (struct CadetOutOfOrderMessage);
-  com->mid = msg->mid;
-  com->env = env;
-  duplicate = GNUNET_NO;
-  GNUNET_CONTAINER_DLL_insert_sorted (struct CadetOutOfOrderMessage,
-                                      is_before,
-                                      &duplicate,
-                                      ccc->head_recv,
-                                      ccc->tail_recv,
-                                      com);
-  ccc->num_recv++;
-  if (GNUNET_YES == duplicate)
-  {
-    /* Duplicate within the queue, drop also (this is not covered by
-       the case above if "delta" >= 64, which could be the case if
-       max_pending_messages is also >= 64 or if our client is unready
-       and we are seeing retransmissions of the message our client is
-       blocked on. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Duplicate payload of %u bytes on %s (mid %u) dropped\n",
-         (unsigned int) payload_size,
-         GCCH_2s (ch),
-         ntohl (msg->mid.mid));
-    GNUNET_STATISTICS_update (stats,
-                              "# duplicate DATA",
-                              1,
-                              GNUNET_NO);
-    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
-                                 ccc->tail_recv,
-                                 com);
-    ccc->num_recv--;
-    GNUNET_MQ_discard (com->env);
-    GNUNET_free (com);
-    send_channel_data_ack (ch);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Queued %s payload of %u bytes on %s-%X(%p) (mid %u, need %u first)\n",
-       (GNUNET_YES == ccc->client_ready)
-       ? "out-of-order"
-       : "client-not-ready",
-       (unsigned int) payload_size,
-       GCCH_2s (ch),
-       ntohl (ccc->ccn.channel_of_client),
-       ccc,
-       ntohl (msg->mid.mid),
-       ntohl (ch->mid_recv.mid));
-  /* NOTE: this ACK we _could_ skip, as the packet is out-of-order and
-     the sender may already be transmitting the previous one.  Needs
-     experimental evaluation to see if/when this ACK helps or
-     hurts. (We might even want another option.) */
-  send_channel_data_ack (ch);
-}
-
-
-/**
- * Function called once the tunnel has sent one of our messages.
- * If the message is unreliable, simply frees the `crm`. If the
- * message was reliable, calculate retransmission time and
- * wait for ACK (or retransmit).
- *
- * @param cls the `struct CadetReliableMessage` that was sent
- * @param cid identifier of the connection within the tunnel, NULL
- *            if transmission failed
- */
-static void
-data_sent_cb (void *cls,
-              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
-
-
-/**
- * We need to retry a transmission, the last one took too long to
- * be acknowledged.
- *
- * @param cls the `struct CadetChannel` where we need to retransmit
- */
-static void
-retry_transmission (void *cls)
-{
-  struct CadetChannel *ch = cls;
-  struct CadetReliableMessage *crm = ch->head_sent;
-
-  ch->retry_data_task = NULL;
-  GNUNET_assert (NULL == crm->qe);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Retrying transmission on %s of message %u\n",
-       GCCH_2s (ch),
-       (unsigned int) ntohl (crm->data_message->mid.mid));
-  crm->qe = GCT_send (ch->t,
-                      &crm->data_message->header,
-                      &data_sent_cb,
-                      crm);
-  GNUNET_assert (NULL == ch->retry_data_task);
-}
-
-
-/**
- * We got an PLAINTEXT_DATA_ACK for a message in our queue, remove it from
- * the queue and tell our client that it can send more.
- *
- * @param ch the channel that got the PLAINTEXT_DATA_ACK
- * @param cti identifier of the connection that delivered the message
- * @param crm the message that got acknowledged
- */
-static void
-handle_matching_ack (struct CadetChannel *ch,
-                     const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
-                     struct CadetReliableMessage *crm)
-{
-  GNUNET_CONTAINER_DLL_remove (ch->head_sent,
-                               ch->tail_sent,
-                               crm);
-  ch->pending_messages--;
-  GNUNET_assert (ch->pending_messages < ch->max_pending_messages);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received DATA_ACK on %s for message %u (%u ACKs pending)\n",
-       GCCH_2s (ch),
-       (unsigned int) ntohl (crm->data_message->mid.mid),
-       ch->pending_messages);
-  if (NULL != crm->qe)
-  {
-    GCT_send_cancel (crm->qe);
-    crm->qe = NULL;
-  }
-  if ( (1 == crm->num_transmissions) &&
-       (NULL != cti) )
-  {
-    GCC_ack_observed (cti);
-    if (0 == memcmp (cti,
-                     &crm->connection_taken,
-                     sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier)))
-    {
-      GCC_latency_observed (cti,
-                            GNUNET_TIME_absolute_get_duration (crm->first_transmission_time));
-    }
-  }
-  GNUNET_free (crm->data_message);
-  GNUNET_free (crm);
-  send_ack_to_client (ch,
-                      (NULL == ch->owner)
-                      ? GNUNET_NO
-                      : GNUNET_YES);
-}
-
-
-/**
- * We got an acknowledgement for payload data for a channel.
- * Possibly resume transmissions.
- *
- * @param ch channel that got the ack
- * @param cti identifier of the connection that delivered the message
- * @param ack details about what was received
- */
-void
-GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
-                                        const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
-                                        const struct GNUNET_CADET_ChannelDataAckMessage *ack)
-{
-  struct CadetReliableMessage *crm;
-  struct CadetReliableMessage *crmn;
-  int found;
-  uint32_t mid_base;
-  uint64_t mid_mask;
-  unsigned int delta;
-
-  GNUNET_break (GNUNET_NO == ch->is_loopback);
-  if (GNUNET_NO == ch->reliable)
-  {
-    /* not expecting ACKs on unreliable channel, odd */
-    GNUNET_break_op (0);
-    return;
-  }
-  /* mid_base is the MID of the next message that the
-     other peer expects (i.e. that is missing!), everything
-     LOWER (but excluding mid_base itself) was received. */
-  mid_base = ntohl (ack->mid.mid);
-  mid_mask = GNUNET_htonll (ack->futures);
-  found = GNUNET_NO;
-  for (crm = ch->head_sent;
-        NULL != crm;
-       crm = crmn)
-  {
-    crmn = crm->next;
-    delta = (unsigned int) (ntohl (crm->data_message->mid.mid) - mid_base);
-    if (delta >= UINT_MAX - ch->max_pending_messages)
-    {
-      /* overflow, means crm was a bit in the past, so this ACK counts for it. */
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Got DATA_ACK with base %u satisfying past message %u on %s\n",
-           (unsigned int) mid_base,
-           ntohl (crm->data_message->mid.mid),
-           GCCH_2s (ch));
-      handle_matching_ack (ch,
-                           cti,
-                           crm);
-      found = GNUNET_YES;
-      continue;
-    }
-    delta--;
-    if (delta >= 64)
-      continue;
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Testing bit %llX for mid %u (base: %u)\n",
-         (1LLU << delta),
-         ntohl (crm->data_message->mid.mid),
-         mid_base);
-    if (0 != (mid_mask & (1LLU << delta)))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Got DATA_ACK with mask for %u on %s\n",
-           ntohl (crm->data_message->mid.mid),
-           GCCH_2s (ch));
-      handle_matching_ack (ch,
-                           cti,
-                           crm);
-      found = GNUNET_YES;
-    }
-  }
-  if (GNUNET_NO == found)
-  {
-    /* ACK for message we already dropped, might have been a
-       duplicate ACK? Ignore. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Duplicate DATA_ACK on %s, ignoring\n",
-         GCCH_2s (ch));
-    GNUNET_STATISTICS_update (stats,
-                              "# duplicate DATA_ACKs",
-                              1,
-                              GNUNET_NO);
-    return;
-  }
-  if (NULL != ch->retry_data_task)
-  {
-    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
-    ch->retry_data_task = NULL;
-  }
-  if ( (NULL != ch->head_sent) &&
-       (NULL == ch->head_sent->qe) )
-    ch->retry_data_task
-      = GNUNET_SCHEDULER_add_at (ch->head_sent->next_retry,
-                                 &retry_transmission,
-                                 ch);
-}
-
-
-/**
- * Destroy channel, based on the other peer closing the
- * connection.  Also needs to remove this channel from
- * the tunnel.
- *
- * @param ch channel to destroy
- * @param cti identifier of the connection that delivered the message,
- *            NULL if we are simulating receiving a destroy due to shutdown
- */
-void
-GCCH_handle_remote_destroy (struct CadetChannel *ch,
-                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti)
-{
-  struct CadetChannelClient *ccc;
-
-  GNUNET_assert (GNUNET_NO == ch->is_loopback);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received remote channel DESTROY for %s\n",
-       GCCH_2s (ch));
-  if (GNUNET_YES == ch->destroy)
-  {
-    /* Local client already gone, this is instant-death. */
-    channel_destroy (ch);
-    return;
-  }
-  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
-  if ( (NULL != ccc) &&
-       (NULL != ccc->head_recv) )
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Lost end of transmission due to remote shutdown on %s\n",
-         GCCH_2s (ch));
-    /* FIXME: change API to notify client about truncated transmission! */
-  }
-  ch->destroy = GNUNET_YES;
-  if (NULL != ccc)
-    GSC_handle_remote_channel_destroy (ccc->c,
-                                       ccc->ccn,
-                                       ch);
-  channel_destroy (ch);
-}
-
-
-/**
- * Test if element @a e1 comes before element @a e2.
- *
- * @param cls closure, to a flag where we indicate duplicate packets
- * @param crm1 an element of to sort
- * @param crm2 another element to sort
- * @return #GNUNET_YES if @e1 < @e2, otherwise #GNUNET_NO
- */
-static int
-cmp_crm_by_next_retry (void *cls,
-                       struct CadetReliableMessage *crm1,
-                       struct CadetReliableMessage *crm2)
-{
-  if (crm1->next_retry.abs_value_us <
-      crm2->next_retry.abs_value_us)
-    return GNUNET_YES;
-  return GNUNET_NO;
-}
-
-
-/**
- * Function called once the tunnel has sent one of our messages.
- * If the message is unreliable, simply frees the `crm`. If the
- * message was reliable, calculate retransmission time and
- * wait for ACK (or retransmit).
- *
- * @param cls the `struct CadetReliableMessage` that was sent
- * @param cid identifier of the connection within the tunnel, NULL
- *            if transmission failed
- */
-static void
-data_sent_cb (void *cls,
-              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
-{
-  struct CadetReliableMessage *crm = cls;
-  struct CadetChannel *ch = crm->ch;
-
-  GNUNET_assert (GNUNET_NO == ch->is_loopback);
-  GNUNET_assert (NULL != crm->qe);
-  crm->qe = NULL;
-  GNUNET_CONTAINER_DLL_remove (ch->head_sent,
-                               ch->tail_sent,
-                               crm);
-  if (GNUNET_NO == ch->reliable)
-  {
-    GNUNET_free (crm->data_message);
-    GNUNET_free (crm);
-    ch->pending_messages--;
-    send_ack_to_client (ch,
-                        (NULL == ch->owner)
-                        ? GNUNET_NO
-                        : GNUNET_YES);
-    return;
-  }
-  if (NULL == cid)
-  {
-    /* There was an error sending. */
-    crm->num_transmissions = GNUNET_SYSERR;
-  }
-  else if (GNUNET_SYSERR != crm->num_transmissions)
-  {
-    /* Increment transmission counter, and possibly store @a cid
-       if this was the first transmission. */
-    crm->num_transmissions++;
-    if (1 == crm->num_transmissions)
-    {
-      crm->first_transmission_time = GNUNET_TIME_absolute_get ();
-      crm->connection_taken = *cid;
-      GCC_ack_expected (cid);
-    }
-  }
-  if ( (0 == crm->retry_delay.rel_value_us) &&
-       (NULL != cid) )
-  {
-    struct CadetConnection *cc = GCC_lookup (cid);
-
-    if (NULL != cc)
-      crm->retry_delay = GCC_get_metrics (cc)->aged_latency;
-    else
-      crm->retry_delay = ch->retry_time;
-  }
-  crm->retry_delay = GNUNET_TIME_STD_BACKOFF (crm->retry_delay);
-  crm->retry_delay = GNUNET_TIME_relative_max (crm->retry_delay,
-                                               MIN_RTT_DELAY);
-  crm->next_retry = GNUNET_TIME_relative_to_absolute (crm->retry_delay);
-
-  GNUNET_CONTAINER_DLL_insert_sorted (struct CadetReliableMessage,
-                                      cmp_crm_by_next_retry,
-                                      NULL,
-                                      ch->head_sent,
-                                      ch->tail_sent,
-                                      crm);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Message %u sent, next transmission on %s in %s\n",
-       (unsigned int) ntohl (crm->data_message->mid.mid),
-       GCCH_2s (ch),
-       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (ch->head_sent->next_retry),
-                                               GNUNET_YES));
-  if (NULL == ch->head_sent->qe)
-  {
-    if (NULL != ch->retry_data_task)
-      GNUNET_SCHEDULER_cancel (ch->retry_data_task);
-    ch->retry_data_task
-      = GNUNET_SCHEDULER_add_at (ch->head_sent->next_retry,
-                                 &retry_transmission,
-                                 ch);
-  }
-}
-
-
-/**
- * Handle data given by a client.
- *
- * Check whether the client is allowed to send in this tunnel, save if
- * channel is reliable and send an ACK to the client if there is still
- * buffer space in the tunnel.
- *
- * @param ch Channel.
- * @param sender_ccn ccn of the sender
- * @param buf payload to transmit.
- * @param buf_len number of bytes in @a buf
- * @return #GNUNET_OK if everything goes well,
- *         #GNUNET_SYSERR in case of an error.
- */
-int
-GCCH_handle_local_data (struct CadetChannel *ch,
-                        struct GNUNET_CADET_ClientChannelNumber sender_ccn,
-                        const char *buf,
-                        size_t buf_len)
-{
-  struct CadetReliableMessage *crm;
-
-  if (ch->pending_messages > ch->max_pending_messages)
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
-  if (GNUNET_YES == ch->destroy)
-  {
-    /* we are going down, drop messages */
-    return GNUNET_OK;
-  }
-  ch->pending_messages++;
-
-  if (GNUNET_YES == ch->is_loopback)
-  {
-    struct CadetChannelClient *receiver;
-    struct GNUNET_MQ_Envelope *env;
-    struct GNUNET_CADET_LocalData *ld;
-    int ack_to_owner;
-
-    env = GNUNET_MQ_msg_extra (ld,
-                               buf_len,
-                               GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
-    if ( (NULL != ch->owner) &&
-         (sender_ccn.channel_of_client ==
-          ch->owner->ccn.channel_of_client) )
-    {
-      receiver = ch->dest;
-      ack_to_owner = GNUNET_YES;
-    }
-    else if ( (NULL != ch->dest) &&
-              (sender_ccn.channel_of_client ==
-               ch->dest->ccn.channel_of_client) )
-    {
-      receiver = ch->owner;
-      ack_to_owner = GNUNET_NO;
-    }
-    else
-    {
-      GNUNET_break (0);
-      return GNUNET_SYSERR;
-    }
-    ld->ccn = receiver->ccn;
-    GNUNET_memcpy (&ld[1],
-                   buf,
-                   buf_len);
-    if (GNUNET_YES == receiver->client_ready)
-    {
-      ch->pending_messages--;
-      GSC_send_to_client (receiver->c,
-                          env);
-      send_ack_to_client (ch,
-                          ack_to_owner);
-    }
-    else
-    {
-      struct CadetOutOfOrderMessage *oom;
-
-      oom = GNUNET_new (struct CadetOutOfOrderMessage);
-      oom->env = env;
-      GNUNET_CONTAINER_DLL_insert_tail (receiver->head_recv,
-                                        receiver->tail_recv,
-                                        oom);
-      receiver->num_recv++;
-    }
-    return GNUNET_OK;
-  }
-
-  /* Everything is correct, send the message. */
-  crm = GNUNET_malloc (sizeof (*crm));
-  crm->ch = ch;
-  crm->data_message = GNUNET_malloc (sizeof (struct GNUNET_CADET_ChannelAppDataMessage)
-                                     + buf_len);
-  crm->data_message->header.size = htons (sizeof (struct GNUNET_CADET_ChannelAppDataMessage) + buf_len);
-  crm->data_message->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA);
-  ch->mid_send.mid = htonl (ntohl (ch->mid_send.mid) + 1);
-  crm->data_message->mid = ch->mid_send;
-  crm->data_message->ctn = ch->ctn;
-  GNUNET_memcpy (&crm->data_message[1],
-                 buf,
-                 buf_len);
-  GNUNET_CONTAINER_DLL_insert_tail (ch->head_sent,
-                                    ch->tail_sent,
-                                    crm);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending message %u from local client to %s with %u bytes\n",
-       ntohl (crm->data_message->mid.mid),
-       GCCH_2s (ch),
-       buf_len);
-  if (NULL != ch->retry_data_task)
-  {
-    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
-    ch->retry_data_task = NULL;
-  }
-  crm->qe = GCT_send (ch->t,
-                      &crm->data_message->header,
-                      &data_sent_cb,
-                      crm);
-  GNUNET_assert (NULL == ch->retry_data_task);
-  return GNUNET_OK;
-}
-
-
-/**
- * Handle ACK from client on local channel.  Means the client is ready
- * for more data, see if we have any for it.
- *
- * @param ch channel to destroy
- * @param client_ccn ccn of the client sending the ack
- */
-void
-GCCH_handle_local_ack (struct CadetChannel *ch,
-                       struct GNUNET_CADET_ClientChannelNumber client_ccn)
-{
-  struct CadetChannelClient *ccc;
-  struct CadetOutOfOrderMessage *com;
-
-  if ( (NULL != ch->owner) &&
-       (ch->owner->ccn.channel_of_client == client_ccn.channel_of_client) )
-    ccc = ch->owner;
-  else if ( (NULL != ch->dest) &&
-            (ch->dest->ccn.channel_of_client == client_ccn.channel_of_client) )
-    ccc = ch->dest;
-  else
-    GNUNET_assert (0);
-  ccc->client_ready = GNUNET_YES;
-  com = ccc->head_recv;
-  if (NULL == com)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got LOCAL_ACK, %s-%X ready to receive more data, but none pending on %s-%X(%p)!\n",
-         GSC_2s (ccc->c),
-         ntohl (client_ccn.channel_of_client),
-         GCCH_2s (ch),
-         ntohl (ccc->ccn.channel_of_client),
-         ccc);
-    return; /* none pending */
-  }
-  if (GNUNET_YES == ch->is_loopback)
-  {
-    int to_owner;
-
-    /* Messages are always in-order, just send */
-    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
-                                 ccc->tail_recv,
-                                 com);
-    ccc->num_recv--;
-    GSC_send_to_client (ccc->c,
-                        com->env);
-    /* Notify sender that we can receive more */
-    if (ccc->ccn.channel_of_client ==
-        ch->owner->ccn.channel_of_client)
-    {
-      to_owner = GNUNET_NO;
-    }
-    else
-    {
-      GNUNET_assert (ccc->ccn.channel_of_client ==
-                     ch->dest->ccn.channel_of_client);
-      to_owner = GNUNET_YES;
-    }
-    send_ack_to_client (ch,
-                        to_owner);
-    GNUNET_free (com);
-    return;
-  }
-
-  if ( (com->mid.mid != ch->mid_recv.mid) &&
-       (GNUNET_NO == ch->out_of_order) &&
-       (GNUNET_YES == ch->reliable) )
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got LOCAL_ACK, %s-%X ready to receive more data (but next one is out-of-order %u vs. %u)!\n",
-         GSC_2s (ccc->c),
-         ntohl (ccc->ccn.channel_of_client),
-         ntohl (com->mid.mid),
-         ntohl (ch->mid_recv.mid));
-    return; /* missing next one in-order */
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got LOCAL_ACK, giving payload message %u to %s-%X on %s\n",
-       ntohl (com->mid.mid),
-       GSC_2s (ccc->c),
-       ntohl (ccc->ccn.channel_of_client),
-       GCCH_2s (ch));
-
-  /* all good, pass next message to client */
-  GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
-                               ccc->tail_recv,
-                               com);
-  ccc->num_recv--;
-  /* FIXME: if unreliable, this is not aggressive
-     enough, as it would be OK to have lost some! */
-
-  ch->mid_recv.mid = htonl (1 + ntohl (com->mid.mid));
-  ch->mid_futures >>= 1; /* equivalent to division by 2 */
-  ccc->client_ready = GNUNET_NO;
-  GSC_send_to_client (ccc->c,
-                      com->env);
-  GNUNET_free (com);
-  send_channel_data_ack (ch);
-  if (NULL != ccc->head_recv)
-    return;
-  if (GNUNET_NO == ch->destroy)
-    return;
-  GCT_send_channel_destroy (ch->t,
-                            ch->ctn);
-  channel_destroy (ch);
-}
-
-
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-chn",__VA_ARGS__)
-
-
-/**
- * Log channel info.
- *
- * @param ch Channel.
- * @param level Debug level to use.
- */
-void
-GCCH_debug (struct CadetChannel *ch,
-            enum GNUNET_ErrorType level)
-{
-  int do_log;
-
-  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
-                                       "cadet-chn",
-                                       __FILE__, __FUNCTION__, __LINE__);
-  if (0 == do_log)
-    return;
-
-  if (NULL == ch)
-  {
-    LOG2 (level, "CHN *** DEBUG NULL CHANNEL ***\n");
-    return;
-  }
-  LOG2 (level,
-        "CHN %s:%X (%p)\n",
-        GCT_2s (ch->t),
-        ch->ctn,
-        ch);
-  if (NULL != ch->owner)
-  {
-    LOG2 (level,
-          "CHN origin %s ready %s local-id: %u\n",
-          GSC_2s (ch->owner->c),
-          ch->owner->client_ready ? "YES" : "NO",
-          ntohl (ch->owner->ccn.channel_of_client));
-  }
-  if (NULL != ch->dest)
-  {
-    LOG2 (level,
-          "CHN destination %s ready %s local-id: %u\n",
-          GSC_2s (ch->dest->c),
-          ch->dest->client_ready ? "YES" : "NO",
-          ntohl (ch->dest->ccn.channel_of_client));
-  }
-  LOG2 (level,
-        "CHN  Message IDs recv: %d (%LLX), send: %d\n",
-        ntohl (ch->mid_recv.mid),
-        (unsigned long long) ch->mid_futures,
-        ntohl (ch->mid_send.mid));
-}
-
-
-
-/* end of gnunet-service-cadet-new_channel.c */
diff --git a/src/cadet/gnunet-service-cadet-new_channel.h b/src/cadet/gnunet-service-cadet-new_channel.h
deleted file mode 100644
index 5167305a6..000000000
--- a/src/cadet/gnunet-service-cadet-new_channel.h
+++ /dev/null
@@ -1,262 +0,0 @@
-
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new_channel.h
- * @brief GNUnet CADET service with encryption
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_CADET_CHANNEL_H
-#define GNUNET_SERVICE_CADET_CHANNEL_H
-
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "cadet_protocol.h"
-
-
-/**
- * A channel is a bidirectional connection between two CADET
- * clients.  Communiation can be reliable, unreliable, in-order
- * or out-of-order.  One client is the "local" client, this
- * one initiated the connection.   The other client is the
- * "incoming" client, this one listened on a port to accept
- * the connection from the "local" client.
- */
-struct CadetChannel;
-
-
-/**
- * Get the static string for identification of the channel.
- *
- * @param ch Channel.
- *
- * @return Static string with the channel IDs.
- */
-const char *
-GCCH_2s (const struct CadetChannel *ch);
-
-
-/**
- * Log channel info.
- *
- * @param ch Channel.
- * @param level Debug level to use.
- */
-void
-GCCH_debug (struct CadetChannel *ch,
-            enum GNUNET_ErrorType level);
-
-
-/**
- * Get the channel's public ID.
- *
- * @param ch Channel.
- *
- * @return ID used to identify the channel with the remote peer.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCCH_get_id (const struct CadetChannel *ch);
-
-
-/**
- * Create a new channel.
- *
- * @param owner local client owning the channel
- * @param owner_id local chid of this channel at the @a owner
- * @param destination peer to which we should build the channel
- * @param port desired port at @a destination
- * @param options options for the channel
- * @return handle to the new channel
- */
-struct CadetChannel *
-GCCH_channel_local_new (struct CadetClient *owner,
-                        struct GNUNET_CADET_ClientChannelNumber owner_id,
-                        struct CadetPeer *destination,
-                        const struct GNUNET_HashCode *port,
-                        uint32_t options);
-
-
-/**
- * A client is bound to the port that we have a channel
- * open to.  Send the acknowledgement for the connection
- * request and establish the link with the client.
- *
- * @param ch open incoming channel
- * @param c client listening on the respective port
- */
-void
-GCCH_bind (struct CadetChannel *ch,
-           struct CadetClient *c);
-
-
-/**
- * Destroy locally created channel.  Called by the
- * local client, so no need to tell the client.
- *
- * @param ch channel to destroy
- * @param c client that caused the destruction
- * @param ccn client number of the client @a c
- */
-void
-GCCH_channel_local_destroy (struct CadetChannel *ch,
-                            struct CadetClient *c,
-                            struct GNUNET_CADET_ClientChannelNumber ccn);
-
-
-/**
- * Function called once and only once after a channel was bound
- * to its tunnel via #GCT_add_channel() is ready for transmission.
- * Note that this is only the case for channels that this peer
- * initiates, as for incoming channels we assume that they are
- * ready for transmission immediately upon receiving the open
- * message.  Used to bootstrap the #GCT_send() process.
- *
- * @param ch the channel for which the tunnel is now ready
- */
-void
-GCCH_tunnel_up (struct CadetChannel *ch);
-
-
-/**
- * Create a new channel based on a request coming in over the network.
- *
- * @param t tunnel to the remote peer
- * @param chid identifier of this channel in the tunnel
- * @param origin peer to who initiated the channel
- * @param port desired local port
- * @param options options for the channel
- * @return handle to the new channel
- */
-struct CadetChannel *
-GCCH_channel_incoming_new (struct CadetTunnel *t,
-                           struct GNUNET_CADET_ChannelTunnelNumber chid,
-                           const struct GNUNET_HashCode *port,
-                           uint32_t options);
-
-
-/**
- * We got a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN message again for
- * this channel.  If the binding was successful, (re)transmit the
- * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
- *
- * @param ch channel that got the duplicate open
- * @param cti identifier of the connection that delivered the message
- */
-void
-GCCH_handle_duplicate_open (struct CadetChannel *ch,
-                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti);
-
-
-
-/**
- * We got payload data for a channel.  Pass it on to the client.
- *
- * @param ch channel that got data
- * @param cti identifier of the connection that delivered the message
- * @param msg message that was received
- */
-void
-GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
-                                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
-                                    const struct GNUNET_CADET_ChannelAppDataMessage *msg);
-
-
-/**
- * We got an acknowledgement for payload data for a channel.
- * Possibly resume transmissions.
- *
- * @param ch channel that got the ack
- * @param cti identifier of the connection that delivered the message
- * @param ack details about what was received
- */
-void
-GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
-                                        const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
-                                        const struct GNUNET_CADET_ChannelDataAckMessage *ack);
-
-
-/**
- * We got an acknowledgement for the creation of the channel
- * (the port is open on the other side). Begin transmissions.
- *
- * @param ch channel to destroy
- * @param cti identifier of the connection that delivered the message,
- *        NULL if the ACK was inferred because we got payload or are on loopback
- */
-void
-GCCH_handle_channel_open_ack (struct CadetChannel *ch,
-                              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti);
-
-
-/**
- * Destroy channel, based on the other peer closing the
- * connection.  Also needs to remove this channel from
- * the tunnel.
- *
- * FIXME: need to make it possible to defer destruction until we have
- * received all messages up to the destroy, and right now the destroy
- * message (and this API) fails to give is the information we need!
- *
- * FIXME: also need to know if the other peer got a destroy from
- * us before!
- *
- * @param ch channel to destroy
- * @param cti identifier of the connection that delivered the message,
- *            NULL during shutdown
- */
-void
-GCCH_handle_remote_destroy (struct CadetChannel *ch,
-                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti);
-
-
-/**
- * Handle data given by a client.
- *
- * Check whether the client is allowed to send in this tunnel, save if
- * channel is reliable and send an ACK to the client if there is still
- * buffer space in the tunnel.
- *
- * @param ch Channel.
- * @param sender_ccn ccn of the sender
- * @param buf payload to transmit.
- * @param buf_len number of bytes in @a buf
- * @return #GNUNET_OK if everything goes well,
- *         #GNUNET_SYSERR in case of an error.
- */
-int
-GCCH_handle_local_data (struct CadetChannel *ch,
-                        struct GNUNET_CADET_ClientChannelNumber sender_ccn,
-                        const char *buf,
-                        size_t buf_len);
-
-
-/**
- * Handle ACK from client on local channel.
- *
- * @param ch channel to destroy
- * @param client_ccn ccn of the client sending the ack
- */
-void
-GCCH_handle_local_ack (struct CadetChannel *ch,
-                       struct GNUNET_CADET_ClientChannelNumber client_ccn);
-
-#endif
diff --git a/src/cadet/gnunet-service-cadet-new_connection.c b/src/cadet/gnunet-service-cadet-new_connection.c
deleted file mode 100644
index 6976e66e4..000000000
--- a/src/cadet/gnunet-service-cadet-new_connection.c
+++ /dev/null
@@ -1,1093 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new_connection.c
- * @brief management of CORE-level end-to-end connections; establishes
- *        end-to-end routes and transmits messages along the route
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- */
-#include "platform.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_channel.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_paths.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-#include "gnunet_cadet_service.h"
-#include "gnunet_statistics_service.h"
-#include "cadet_protocol.h"
-
-
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-con",__VA_ARGS__)
-
-
-/**
- * All the states a connection can be in.
- */
-enum CadetConnectionState
-{
-  /**
-   * Uninitialized status, we have not yet even gotten the message queue.
-   */
-  CADET_CONNECTION_NEW,
-
-  /**
-   * Connection create message in queue, awaiting transmission by CORE.
-   */
-  CADET_CONNECTION_SENDING_CREATE,
-
-  /**
-   * Connection create message sent, waiting for ACK.
-   */
-  CADET_CONNECTION_SENT,
-
-  /**
-   * We are an inbound connection, and received a CREATE. Need to
-   * send an CREATE_ACK back.
-   */
-  CADET_CONNECTION_CREATE_RECEIVED,
-
-  /**
-   * Connection confirmed, ready to carry traffic.
-   */
-  CADET_CONNECTION_READY
-
-};
-
-
-/**
- * Low-level connection to a destination.
- */
-struct CadetConnection
-{
-
-  /**
-   * ID of the connection.
-   */
-  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
-
-  /**
-   * To which peer does this connection go?
-   */
-  struct CadetPeer *destination;
-
-  /**
-   * Which tunnel is using this connection?
-   */
-  struct CadetTConnection *ct;
-
-  /**
-   * Path we are using to our destination.
-   */
-  struct CadetPeerPath *path;
-
-  /**
-   * Pending message, NULL if we are ready to transmit.
-   */
-  struct GNUNET_MQ_Envelope *env;
-
-  /**
-   * Handle for calling #GCP_request_mq_cancel() once we are finished.
-   */
-  struct GCP_MessageQueueManager *mq_man;
-
-  /**
-   * Task for connection maintenance.
-   */
-  struct GNUNET_SCHEDULER_Task *task;
-
-  /**
-   * Queue entry for keepalive messages.
-   */
-  struct CadetTunnelQueueEntry *keepalive_qe;
-
-  /**
-   * Function to call once we are ready to transmit.
-   */
-  GCC_ReadyCallback ready_cb;
-
-  /**
-   * Closure for @e ready_cb.
-   */
-  void *ready_cb_cls;
-
-  /**
-   * How long do we wait before we try again with a CREATE message?
-   */
-  struct GNUNET_TIME_Relative retry_delay;
-
-  /**
-   * Performance metrics for this connection.
-   */
-  struct CadetConnectionMetrics metrics;
-
-  /**
-   * State of the connection.
-   */
-  enum CadetConnectionState state;
-
-  /**
-   * Options for the route, control buffering.
-   */
-  enum GNUNET_CADET_ChannelOption options;
-
-  /**
-   * How many latency observations did we make for this connection?
-   */
-  unsigned int latency_datapoints;
-
-  /**
-   * Offset of our @e destination in @e path.
-   */
-  unsigned int off;
-
-  /**
-   * Are we ready to transmit via @e mq_man right now?
-   */
-  int mqm_ready;
-
-};
-
-
-/**
- * Lookup a connection by its identifier.
- *
- * @param cid identifier to resolve
- * @return NULL if connection was not found
- */
-struct CadetConnection *
-GCC_lookup (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
-{
-  return GNUNET_CONTAINER_multishortmap_get (connections,
-                                             &cid->connection_of_tunnel);
-}
-
-
-/**
- * Update the connection state. Also triggers the necessary
- * MQM notifications.
- *
- * @param cc connection to update the state for
- * @param new_state new state for @a cc
- * @param new_mqm_ready new `mqm_ready` state for @a cc
- */
-static void
-update_state (struct CadetConnection *cc,
-              enum CadetConnectionState new_state,
-              int new_mqm_ready)
-{
-  int old_ready;
-  int new_ready;
-
-  if ( (new_state == cc->state) &&
-       (new_mqm_ready == cc->mqm_ready) )
-    return; /* no change, nothing to do */
-  old_ready = ( (CADET_CONNECTION_READY == cc->state) &&
-                (GNUNET_YES == cc->mqm_ready) );
-  new_ready = ( (CADET_CONNECTION_READY == new_state) &&
-                (GNUNET_YES == new_mqm_ready) );
-  cc->state = new_state;
-  cc->mqm_ready = new_mqm_ready;
-  if (old_ready != new_ready)
-    cc->ready_cb (cc->ready_cb_cls,
-                  new_ready);
-}
-
-
-/**
- * Destroy a connection, part of the internal implementation.  Called
- * only from #GCC_destroy_from_core() or #GCC_destroy_from_tunnel().
- *
- * @param cc connection to destroy
- */
-static void
-GCC_destroy (struct CadetConnection *cc)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying %s\n",
-       GCC_2s (cc));
-  if (NULL != cc->mq_man)
-  {
-    GCP_request_mq_cancel (cc->mq_man,
-                           NULL);
-    cc->mq_man = NULL;
-  }
-  if (NULL != cc->task)
-  {
-    GNUNET_SCHEDULER_cancel (cc->task);
-    cc->task = NULL;
-  }
-  if (NULL != cc->keepalive_qe)
-  {
-    GCT_send_cancel (cc->keepalive_qe);
-    cc->keepalive_qe = NULL;
-  }
-  GCPP_del_connection (cc->path,
-                       cc->off,
-                       cc);
-  for (unsigned int i=0;i<cc->off;i++)
-    GCP_remove_connection (GCPP_get_peer_at_offset (cc->path,
-                                                    i),
-                           cc);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multishortmap_remove (connections,
-                                                        &GCC_get_id (cc)->connection_of_tunnel,
-                                                        cc));
-  GNUNET_free (cc);
-}
-
-
-
-/**
- * Destroy a connection, called when the CORE layer is already done
- * (i.e. has received a BROKEN message), but if we still have to
- * communicate the destruction of the connection to the tunnel (if one
- * exists).
- *
- * @param cc connection to destroy
- */
-void
-GCC_destroy_without_core (struct CadetConnection *cc)
-{
-  if (NULL != cc->ct)
-  {
-    GCT_connection_lost (cc->ct);
-    cc->ct = NULL;
-  }
-  GCC_destroy (cc);
-}
-
-
-/**
- * Destroy a connection, called if the tunnel association with the
- * connection was already broken, but we still need to notify the CORE
- * layer about the breakage.
- *
- * @param cc connection to destroy
- */
-void
-GCC_destroy_without_tunnel (struct CadetConnection *cc)
-{
-  cc->ct = NULL;
-  if ( (CADET_CONNECTION_SENDING_CREATE != cc->state) &&
-       (NULL != cc->mq_man) )
-  {
-    struct GNUNET_MQ_Envelope *env;
-    struct GNUNET_CADET_ConnectionDestroyMessage *destroy_msg;
-
-    /* Need to notify next hop that we are down. */
-    env = GNUNET_MQ_msg (destroy_msg,
-                         GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY);
-    destroy_msg->cid = cc->cid;
-    GCP_request_mq_cancel (cc->mq_man,
-                           env);
-    cc->mq_man = NULL;
-  }
-  GCC_destroy (cc);
-}
-
-
-/**
- * Return the tunnel associated with this connection.
- *
- * @param cc connection to query
- * @return corresponding entry in the tunnel's connection list
- */
-struct CadetTConnection *
-GCC_get_ct (struct CadetConnection *cc)
-{
-  return cc->ct;
-}
-
-
-/**
- * Obtain performance @a metrics from @a cc.
- *
- * @param cc connection to query
- * @return the metrics
- */
-const struct CadetConnectionMetrics *
-GCC_get_metrics (struct CadetConnection *cc)
-{
-  return &cc->metrics;
-}
-
-
-/**
- * Send a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE through the
- * tunnel to prevent it from timing out.
- *
- * @param cls the `struct CadetConnection` to keep alive.
- */
-static void
-send_keepalive (void *cls);
-
-
-/**
- * Keepalive was transmitted.  Remember this, and possibly
- * schedule the next one.
- *
- * @param cls the `struct CadetConnection` to keep alive.
- * @param cid identifier of the connection within the tunnel, NULL
- *            if transmission failed
- */
-static void
-keepalive_done (void *cls,
-                const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
-{
-  struct CadetConnection *cc = cls;
-
-  cc->keepalive_qe = NULL;
-  if ( (GNUNET_YES == cc->mqm_ready) &&
-       (NULL == cc->task) )
-    cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
-                                             &send_keepalive,
-                                             cc);
-}
-
-
-/**
- * Send a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE through the
- * tunnel to prevent it from timing out.
- *
- * @param cls the `struct CadetConnection` to keep alive.
- */
-static void
-send_keepalive (void *cls)
-{
-  struct CadetConnection *cc = cls;
-  struct GNUNET_MessageHeader msg;
-
-  cc->task = NULL;
-  if (CADET_TUNNEL_KEY_OK != GCT_get_estate (cc->ct->t))
-  {
-    /* Tunnel not yet ready, wait with keepalives... */
-    cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
-                                             &send_keepalive,
-                                             cc);
-    return;
-  }
-  GNUNET_assert (NULL != cc->ct);
-  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
-  GNUNET_assert (NULL == cc->keepalive_qe);
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "Sending KEEPALIVE on behalf of %s via %s\n",
-       GCC_2s (cc),
-       GCT_2s (cc->ct->t));
-  GNUNET_STATISTICS_update (stats,
-                            "# keepalives sent",
-                            1,
-                            GNUNET_NO);
-  msg.size = htons (sizeof (msg));
-  msg.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE);
-
-  cc->keepalive_qe
-    = GCT_send (cc->ct->t,
-                &msg,
-                &keepalive_done,
-                cc);
-}
-
-
-/**
- * We sent a message for which we expect to receive an ACK via
- * the connection identified by @a cti.
- *
- * @param cid connection identifier where we expect an ACK
- */
-void
-GCC_ack_expected (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
-{
-  struct CadetConnection *cc;
-
-  cc = GCC_lookup (cid);
-  if (NULL == cc)
-    return; /* whopise, connection alredy down? */
-  cc->metrics.num_acked_transmissions++;
-}
-
-
-/**
- * We observed an ACK for a message that was originally sent via
- * the connection identified by @a cti.
- *
- * @param cti connection identifier where we got an ACK for a message
- *            that was originally sent via this connection (the ACK
- *            may have gotten back to us via a different connection).
- */
-void
-GCC_ack_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
-{
-  struct CadetConnection *cc;
-
-  cc = GCC_lookup (cid);
-  if (NULL == cc)
-    return; /* whopise, connection alredy down? */
-  cc->metrics.num_successes++;
-}
-
-
-/**
- * We observed some the given @a latency on the connection
- * identified by @a cti.  (The same connection was taken
- * in both directions.)
- *
- * @param cid connection identifier where we measured latency
- * @param latency the observed latency
- */
-void
-GCC_latency_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-                      struct GNUNET_TIME_Relative latency)
-{
-  struct CadetConnection *cc;
-  double weight;
-  double result;
-
-  cc = GCC_lookup (cid);
-  if (NULL == cc)
-    return; /* whopise, connection alredy down? */
-  GNUNET_STATISTICS_update (stats,
-                            "# latencies observed",
-                            1,
-                            GNUNET_NO);
-  cc->latency_datapoints++;
-  if (cc->latency_datapoints >= 7)
-    weight = 7.0;
-  else
-    weight = cc->latency_datapoints;
-  /* Compute weighted average, giving at MOST weight 7 to the
-     existing values, or less if that value is based on fewer than 7
-     measurements. */
-  result = (weight * cc->metrics.aged_latency.rel_value_us) + 1.0 * latency.rel_value_us;
-  result /= (weight + 1.0);
-  cc->metrics.aged_latency.rel_value_us = (uint64_t) result;
-}
-
-
-/**
- * A #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK was received for this connection, implying
- * that the end-to-end connection is up.  Process it.
- *
- * @param cc the connection that got the ACK.
- */
-void
-GCC_handle_connection_create_ack (struct CadetConnection *cc)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received CADET_CONNECTION_CREATE_ACK for %s in state %d (%s)\n",
-       GCC_2s (cc),
-       cc->state,
-       (GNUNET_YES == cc->mqm_ready) ? "MQM ready" : "MQM busy");
-  if (CADET_CONNECTION_READY == cc->state)
-    return; /* Duplicate ACK, ignore */
-  if (NULL != cc->task)
-  {
-    GNUNET_SCHEDULER_cancel (cc->task);
-    cc->task = NULL;
-  }
-  cc->metrics.age = GNUNET_TIME_absolute_get ();
-  update_state (cc,
-                CADET_CONNECTION_READY,
-                cc->mqm_ready);
-  if ( (NULL == cc->keepalive_qe) &&
-       (GNUNET_YES == cc->mqm_ready) &&
-       (NULL == cc->task) )
-    cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
-                                             &send_keepalive,
-                                             cc);
-}
-
-
-/**
- * Handle KX message.
- *
- * @param cc connection that received encrypted message
- * @param msg the key exchange message
- */
-void
-GCC_handle_kx (struct CadetConnection *cc,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
-{
-  if (CADET_CONNECTION_SENT == cc->state)
-  {
-    /* We didn't get the CADET_CONNECTION_CREATE_ACK, but instead got payload. That's fine,
-       clearly something is working, so pretend we got an ACK. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Faking connection CADET_CONNECTION_CREATE_ACK for %s due to KX\n",
-         GCC_2s (cc));
-    GCC_handle_connection_create_ack (cc);
-  }
-  GCT_handle_kx (cc->ct,
-                 msg);
-}
-
-
-/**
- * Handle KX_AUTH message.
- *
- * @param cc connection that received encrypted message
- * @param msg the key exchange message
- */
-void
-GCC_handle_kx_auth (struct CadetConnection *cc,
-                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg)
-{
-  if (CADET_CONNECTION_SENT == cc->state)
-  {
-    /* We didn't get the CADET_CONNECTION_CREATE_ACK, but instead got payload. That's fine,
-       clearly something is working, so pretend we got an ACK. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Faking connection CADET_CONNECTION_CREATE_ACK for %s due to KX\n",
-         GCC_2s (cc));
-    GCC_handle_connection_create_ack (cc);
-  }
-  GCT_handle_kx_auth (cc->ct,
-                      msg);
-}
-
-
-/**
- * Handle encrypted message.
- *
- * @param cc connection that received encrypted message
- * @param msg the encrypted message to decrypt
- */
-void
-GCC_handle_encrypted (struct CadetConnection *cc,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
-{
-  if (CADET_CONNECTION_SENT == cc->state)
-  {
-    /* We didn't get the CREATE_ACK, but instead got payload. That's fine,
-       clearly something is working, so pretend we got an ACK. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Faking connection ACK for %s due to ENCRYPTED payload\n",
-         GCC_2s (cc));
-    GCC_handle_connection_create_ack (cc);
-  }
-  cc->metrics.last_use = GNUNET_TIME_absolute_get ();
-  GCT_handle_encrypted (cc->ct,
-                        msg);
-}
-
-
-/**
- * Send a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE message to the
- * first hop.
- *
- * @param cls the `struct CadetConnection` to initiate
- */
-static void
-send_create (void *cls)
-{
-  struct CadetConnection *cc = cls;
-  struct GNUNET_CADET_ConnectionCreateMessage *create_msg;
-  struct GNUNET_PeerIdentity *pids;
-  struct GNUNET_MQ_Envelope *env;
-  unsigned int path_length;
-
-  cc->task = NULL;
-  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
-  path_length = GCPP_get_length (cc->path);
-  env = GNUNET_MQ_msg_extra (create_msg,
-                             (1 + path_length) * sizeof (struct GNUNET_PeerIdentity),
-                             GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE);
-  create_msg->options = htonl ((uint32_t) cc->options);
-  create_msg->cid = cc->cid;
-  pids = (struct GNUNET_PeerIdentity *) &create_msg[1];
-  pids[0] = my_full_id;
-  for (unsigned int i=0;i<path_length;i++)
-    pids[i + 1] = *GCP_get_id (GCPP_get_peer_at_offset (cc->path,
-                                                        i));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending CADET_CONNECTION_CREATE message for %s\n",
-       GCC_2s (cc));
-  cc->env = env;
-  update_state (cc,
-                CADET_CONNECTION_SENT,
-                GNUNET_NO);
-  GCP_send (cc->mq_man,
-            env);
-}
-
-
-/**
- * Send a CREATE_ACK message towards the origin.
- *
- * @param cls the `struct CadetConnection` to initiate
- */
-static void
-send_create_ack (void *cls)
-{
-  struct CadetConnection *cc = cls;
-  struct GNUNET_CADET_ConnectionCreateAckMessage *ack_msg;
-  struct GNUNET_MQ_Envelope *env;
-
-  cc->task = NULL;
-  GNUNET_assert (CADET_CONNECTION_CREATE_RECEIVED == cc->state);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending CONNECTION_CREATE_ACK message for %s\n",
-       GCC_2s (cc));
-  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
-  env = GNUNET_MQ_msg (ack_msg,
-                       GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK);
-  ack_msg->cid = cc->cid;
-  cc->env = env;
-  update_state (cc,
-                CADET_CONNECTION_READY,
-                GNUNET_NO);
-  GCP_send (cc->mq_man,
-            env);
-}
-
-
-/**
- * We got a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE for a
- * connection that we already have.  Either our ACK got lost
- * or something is fishy.  Consider retransmitting the ACK.
- *
- * @param cc connection that got the duplicate CREATE
- */
-void
-GCC_handle_duplicate_create (struct CadetConnection *cc)
-{
-  if (GNUNET_YES == cc->mqm_ready)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got duplicate CREATE for %s, scheduling another ACK (%s)\n",
-         GCC_2s (cc),
-         (GNUNET_YES == cc->mqm_ready) ? "MQM ready" : "MQM busy");
-    /* Revert back to the state of having only received the 'CREATE',
-       and immediately proceed to send the CREATE_ACK. */
-    update_state (cc,
-                  CADET_CONNECTION_CREATE_RECEIVED,
-                  cc->mqm_ready);
-    if (NULL != cc->task)
-      GNUNET_SCHEDULER_cancel (cc->task);
-    cc->task = GNUNET_SCHEDULER_add_now (&send_create_ack,
-                                         cc);
-  }
-  else
-  {
-    /* We are currently sending something else back, which
-       can only be an ACK or payload, either of which would
-       do. So actually no need to do anything. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got duplicate CREATE for %s. MQ is busy, not queueing another ACK\n",
-         GCC_2s (cc));
-  }
-}
-
-
-/**
- * There has been a change in the message queue existence for our
- * peer at the first hop.  Adjust accordingly.
- *
- * @param cls the `struct CadetConnection`
- * @param available #GNUNET_YES if sending is now possible,
- *                  #GNUNET_NO if sending is no longer possible
- *                  #GNUNET_SYSERR if sending is no longer possible
- *                                 and the last envelope was discarded
- */
-static void
-manage_first_hop_mq (void *cls,
-                     int available)
-{
-  struct CadetConnection *cc = cls;
-
-  if (GNUNET_YES != available)
-  {
-    /* Connection is down, for now... */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Core MQ for %s went down\n",
-         GCC_2s (cc));
-    update_state (cc,
-                  CADET_CONNECTION_NEW,
-                  GNUNET_NO);
-    cc->retry_delay = GNUNET_TIME_UNIT_ZERO;
-    if (NULL != cc->task)
-    {
-      GNUNET_SCHEDULER_cancel (cc->task);
-      cc->task = NULL;
-    }
-    return;
-  }
-
-  update_state (cc,
-                cc->state,
-                GNUNET_YES);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Core MQ for %s became available in state %d\n",
-       GCC_2s (cc),
-       cc->state);
-  switch (cc->state)
-  {
-  case CADET_CONNECTION_NEW:
-    /* Transmit immediately */
-    cc->task = GNUNET_SCHEDULER_add_now (&send_create,
-                                         cc);
-    break;
-  case CADET_CONNECTION_SENDING_CREATE:
-    /* Should not be possible to be called in this state. */
-    GNUNET_assert (0);
-    break;
-  case CADET_CONNECTION_SENT:
-    /* Retry a bit later... */
-    cc->retry_delay = GNUNET_TIME_STD_BACKOFF (cc->retry_delay);
-    cc->task = GNUNET_SCHEDULER_add_delayed (cc->retry_delay,
-                                             &send_create,
-                                             cc);
-    break;
-  case CADET_CONNECTION_CREATE_RECEIVED:
-    /* We got the 'CREATE' (incoming connection), should send the CREATE_ACK */
-    cc->metrics.age = GNUNET_TIME_absolute_get ();
-    cc->task = GNUNET_SCHEDULER_add_now (&send_create_ack,
-                                         cc);
-    break;
-  case CADET_CONNECTION_READY:
-    if ( (NULL == cc->keepalive_qe) &&
-         (GNUNET_YES == cc->mqm_ready) &&
-         (NULL == cc->task) )
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Scheduling keepalive for %s in %s\n",
-           GCC_2s (cc),
-           GNUNET_STRINGS_relative_time_to_string (keepalive_period,
-                                                   GNUNET_YES));
-      cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
-                                               &send_keepalive,
-                                               cc);
-    }
-    break;
-  }
-}
-
-
-/**
- * Create a connection to @a destination via @a path and notify @a cb
- * whenever we are ready for more data.  Shared logic independent of
- * who is initiating the connection.
- *
- * @param destination where to go
- * @param path which path to take (may not be the full path)
- * @param off offset of @a destination on @a path
- * @param options options for the connection
- * @param ct which tunnel uses this connection
- * @param init_state initial state for the connection
- * @param ready_cb function to call when ready to transmit
- * @param ready_cb_cls closure for @a cb
- * @return handle to the connection
- */
-static struct CadetConnection *
-connection_create (struct CadetPeer *destination,
-                   struct CadetPeerPath *path,
-                   unsigned int off,
-                   enum GNUNET_CADET_ChannelOption options,
-                   struct CadetTConnection *ct,
-                   const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-                   enum CadetConnectionState init_state,
-                   GCC_ReadyCallback ready_cb,
-                   void *ready_cb_cls)
-{
-  struct CadetConnection *cc;
-  struct CadetPeer *first_hop;
-
-  cc = GNUNET_new (struct CadetConnection);
-  cc->options = options;
-  cc->state = init_state;
-  cc->ct = ct;
-  cc->cid = *cid;
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multishortmap_put (connections,
-                                                     &GCC_get_id (cc)->connection_of_tunnel,
-                                                     cc,
-                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  cc->ready_cb = ready_cb;
-  cc->ready_cb_cls = ready_cb_cls;
-  cc->path = path;
-  cc->off = off;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating %s using path %s\n",
-       GCC_2s (cc),
-       GCPP_2s (path));
-  GCPP_add_connection (path,
-                       off,
-                       cc);
-  for (unsigned int i=0;i<off;i++)
-    GCP_add_connection (GCPP_get_peer_at_offset (path,
-                                                 i),
-                        cc);
-
-  first_hop = GCPP_get_peer_at_offset (path,
-                                       0);
-  cc->mq_man = GCP_request_mq (first_hop,
-                               &manage_first_hop_mq,
-                               cc);
-  return cc;
-}
-
-
-/**
- * Create a connection to @a destination via @a path and
- * notify @a cb whenever we are ready for more data.  This
- * is an inbound tunnel, so we must use the existing @a cid
- *
- * @param destination where to go
- * @param path which path to take (may not be the full path)
- * @param options options for the connection
- * @param ct which tunnel uses this connection
- * @param ready_cb function to call when ready to transmit
- * @param ready_cb_cls closure for @a cb
- * @return handle to the connection, NULL if we already have
- *         a connection that takes precedence on @a path
- */
-struct CadetConnection *
-GCC_create_inbound (struct CadetPeer *destination,
-                    struct CadetPeerPath *path,
-                    enum GNUNET_CADET_ChannelOption options,
-                    struct CadetTConnection *ct,
-                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-                    GCC_ReadyCallback ready_cb,
-                    void *ready_cb_cls)
-{
-  struct CadetConnection *cc;
-  unsigned int off;
-
-  off = GCPP_find_peer (path,
-                        destination);
-  GNUNET_assert (UINT_MAX != off);
-  cc = GCPP_get_connection (path,
-                            destination,
-                            off);
-  if (NULL != cc)
-  {
-    int cmp;
-
-    cmp = memcmp (cid,
-                  &cc->cid,
-                  sizeof (*cid));
-    if (0 == cmp)
-    {
-      /* Two peers picked the SAME random connection identifier at the
-         same time for the same path? Must be malicious.  Drop
-         connection (existing and inbound), even if it is the only
-         one. */
-      GNUNET_break_op (0);
-      GCT_connection_lost (cc->ct);
-      GCC_destroy_without_tunnel (cc);
-      return NULL;
-    }
-    if (0 < cmp)
-    {
-      /* drop existing */
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Got two connections on %s, dropping my existing %s\n",
-           GCPP_2s (path),
-           GCC_2s (cc));
-      GCT_connection_lost (cc->ct);
-      GCC_destroy_without_tunnel (cc);
-    }
-    else
-    {
-      /* keep existing */
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Got two connections on %s, keeping my existing %s\n",
-           GCPP_2s (path),
-           GCC_2s (cc));
-      return NULL;
-    }
-  }
-
-  return connection_create (destination,
-                            path,
-                            off,
-                            options,
-                            ct,
-                            cid,
-                            CADET_CONNECTION_CREATE_RECEIVED,
-                            ready_cb,
-                            ready_cb_cls);
-}
-
-
-/**
- * Create a connection to @a destination via @a path and
- * notify @a cb whenever we are ready for more data.
- *
- * @param destination where to go
- * @param path which path to take (may not be the full path)
- * @param off offset of @a destination on @a path
- * @param options options for the connection
- * @param ct tunnel that uses the connection
- * @param ready_cb function to call when ready to transmit
- * @param ready_cb_cls closure for @a cb
- * @return handle to the connection
- */
-struct CadetConnection *
-GCC_create (struct CadetPeer *destination,
-            struct CadetPeerPath *path,
-            unsigned int off,
-            enum GNUNET_CADET_ChannelOption options,
-            struct CadetTConnection *ct,
-            GCC_ReadyCallback ready_cb,
-            void *ready_cb_cls)
-{
-  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
-
-  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_NONCE,
-                              &cid,
-                              sizeof (cid));
-  return connection_create (destination,
-                            path,
-                            off,
-                            options,
-                            ct,
-                            &cid,
-                            CADET_CONNECTION_NEW,
-                            ready_cb,
-                            ready_cb_cls);
-}
-
-
-/**
- * Transmit message @a msg via connection @a cc.  Must only be called
- * (once) after the connection has signalled that it is ready via the
- * `ready_cb`.  Clients can also use #GCC_is_ready() to check if the
- * connection is right now ready for transmission.
- *
- * @param cc connection identification
- * @param env envelope with message to transmit; must NOT
- *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
- */
-void
-GCC_transmit (struct CadetConnection *cc,
-              struct GNUNET_MQ_Envelope *env)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Scheduling message for transmission on %s\n",
-       GCC_2s (cc));
-  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
-  GNUNET_assert (CADET_CONNECTION_READY == cc->state);
-  cc->metrics.last_use = GNUNET_TIME_absolute_get ();
-  cc->mqm_ready = GNUNET_NO;
-  if (NULL != cc->task)
-  {
-    GNUNET_SCHEDULER_cancel (cc->task);
-    cc->task = NULL;
-  }
-  GCP_send (cc->mq_man,
-            env);
-}
-
-
-/**
- * Obtain the path used by this connection.
- *
- * @param cc connection
- * @return path to @a cc
- */
-struct CadetPeerPath *
-GCC_get_path (struct CadetConnection *cc)
-{
-  return cc->path;
-}
-
-
-/**
- * Obtain unique ID for the connection.
- *
- * @param cc connection.
- * @return unique number of the connection
- */
-const struct GNUNET_CADET_ConnectionTunnelIdentifier *
-GCC_get_id (struct CadetConnection *cc)
-{
-  return &cc->cid;
-}
-
-
-/**
- * Get a (static) string for a connection.
- *
- * @param cc Connection.
- */
-const char *
-GCC_2s (const struct CadetConnection *cc)
-{
-  static char buf[128];
-
-  if (NULL == cc)
-    return "Connection(NULL)";
-
-  if (NULL != cc->ct)
-  {
-    GNUNET_snprintf (buf,
-                     sizeof (buf),
-                     "Connection %s (%s)",
-                     GNUNET_sh2s (&cc->cid.connection_of_tunnel),
-                     GCT_2s (cc->ct->t));
-    return buf;
-  }
-  GNUNET_snprintf (buf,
-                   sizeof (buf),
-                   "Connection %s",
-                   GNUNET_sh2s (&cc->cid.connection_of_tunnel));
-  return buf;
-}
-
-
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-con",__VA_ARGS__)
-
-
-/**
- * Log connection info.
- *
- * @param cc connection
- * @param level Debug level to use.
- */
-void
-GCC_debug (struct CadetConnection *cc,
-           enum GNUNET_ErrorType level)
-{
-  int do_log;
-
-  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
-                                       "cadet-con",
-                                       __FILE__, __FUNCTION__, __LINE__);
-  if (0 == do_log)
-    return;
-  if (NULL == cc)
-  {
-    LOG2 (level,
-          "Connection (NULL)\n");
-    return;
-  }
-  LOG2 (level,
-        "%s to %s via path %s in state %d is %s\n",
-        GCC_2s (cc),
-        GCP_2s (cc->destination),
-        GCPP_2s (cc->path),
-        cc->state,
-        (GNUNET_YES == cc->mqm_ready) ? "ready" : "busy");
-}
-
-/* end of gnunet-service-cadet-new_connection.c */
diff --git a/src/cadet/gnunet-service-cadet-new_connection.h b/src/cadet/gnunet-service-cadet-new_connection.h
deleted file mode 100644
index e48b208fd..000000000
--- a/src/cadet/gnunet-service-cadet-new_connection.h
+++ /dev/null
@@ -1,339 +0,0 @@
-
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new_connection.h
- * @brief A connection is a live end-to-end messaging mechanism
- *       where the peers are identified by a path and know how
- *       to forward along the route using a connection identifier
- *       for routing the data.
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_CADET_CONNECTION_H
-#define GNUNET_SERVICE_CADET_CONNECTION_H
-
-#include "gnunet_util_lib.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "cadet_protocol.h"
-
-
-/**
- * Function called to notify tunnel about change in our readyness.
- *
- * @param cls closure
- * @param is_ready #GNUNET_YES if the connection is now ready for transmission,
- *                 #GNUNET_NO if the connection is no longer ready for transmission
- */
-typedef void
-(*GCC_ReadyCallback)(void *cls,
-                     int is_ready);
-
-
-/**
- * Destroy a connection, called when the CORE layer is already done
- * (i.e. has received a BROKEN message), but if we still have to
- * communicate the destruction of the connection to the tunnel (if one
- * exists).
- *
- * @param cc connection to destroy
- */
-void
-GCC_destroy_without_core (struct CadetConnection *cc);
-
-
-/**
- * Destroy a connection, called if the tunnel association with the
- * connection was already broken, but we still need to notify the CORE
- * layer about the breakage.
- *
- * @param cc connection to destroy
- */
-void
-GCC_destroy_without_tunnel (struct CadetConnection *cc);
-
-
-/**
- * Lookup a connection by its identifier.
- *
- * @param cid identifier to resolve
- * @return NULL if connection was not found
- */
-struct CadetConnection *
-GCC_lookup (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
-
-
-/**
- * Create a connection to @a destination via @a path and
- * notify @a cb whenever we are ready for more data.
- *
- * @param destination where to go
- * @param path which path to take (may not be the full path)
- * @param off offset of @a destination on @a path
- * @param options options for the connection
- * @param ct which tunnel uses this connection
- * @param ready_cb function to call when ready to transmit
- * @param ready_cb_cls closure for @a cb
- * @return handle to the connection
- */
-struct CadetConnection *
-GCC_create (struct CadetPeer *destination,
-            struct CadetPeerPath *path,
-            unsigned int off,
-            enum GNUNET_CADET_ChannelOption options,
-            struct CadetTConnection *ct,
-            GCC_ReadyCallback ready_cb,
-            void *ready_cb_cls);
-
-
-/**
- * Create a connection to @a destination via @a path and
- * notify @a cb whenever we are ready for more data.  This
- * is an inbound tunnel, so we must use the existing @a cid
- *
- * @param destination where to go
- * @param path which path to take (may not be the full path)
- * @param options options for the connection
- * @param ct which tunnel uses this connection
- * @param ready_cb function to call when ready to transmit
- * @param ready_cb_cls closure for @a cb
- * @return handle to the connection, NULL if we already have
- *         a connection that takes precedence on @a path
- */
-struct CadetConnection *
-GCC_create_inbound (struct CadetPeer *destination,
-                    struct CadetPeerPath *path,
-                    enum GNUNET_CADET_ChannelOption options,
-                    struct CadetTConnection *ct,
-                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-                    GCC_ReadyCallback ready_cb,
-                    void *ready_cb_cls);
-
-
-/**
- * Transmit message @a msg via connection @a cc.  Must only be called
- * (once) after the connection has signalled that it is ready via the
- * `ready_cb`.  Clients can also use #GCC_is_ready() to check if the
- * connection is right now ready for transmission.
- *
- * @param cc connection identification
- * @param env envelope with message to transmit;
- *            the #GNUNET_MQ_notify_send() must not have yet been used
- *            for the envelope.  Also, the message better match the
- *            connection identifier of this connection...
- */
-void
-GCC_transmit (struct CadetConnection *cc,
-              struct GNUNET_MQ_Envelope *env);
-
-
-/**
- * A CREATE_ACK was received for this connection, process it.
- *
- * @param cc the connection that got the ACK.
- */
-void
-GCC_handle_connection_create_ack (struct CadetConnection *cc);
-
-
-/**
- * We got a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE for a
- * connection that we already have.  Either our ACK got lost
- * or something is fishy.  Consider retransmitting the ACK.
- *
- * @param cc connection that got the duplicate CREATE
- */
-void
-GCC_handle_duplicate_create (struct CadetConnection *cc);
-
-
-/**
- * Handle KX message.
- *
- * @param cc connection that received encrypted message
- * @param msg the key exchange message
- */
-void
-GCC_handle_kx (struct CadetConnection *cc,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg);
-
-
-/**
- * Handle KX_AUTH message.
- *
- * @param cc connection that received encrypted message
- * @param msg the key exchange message
- */
-void
-GCC_handle_kx_auth (struct CadetConnection *cc,
-                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg);
-
-
-/**
- * Performance metrics for a connection.
- */
-struct CadetConnectionMetrics
-{
-
-  /**
-   * Our current best estimate of the latency, based on a weighted
-   * average of at least @a latency_datapoints values.
-   */
-  struct GNUNET_TIME_Relative aged_latency;
-
-  /**
-   * When was this connection first established? (by us sending or
-   * receiving the CREATE_ACK for the first time)
-   */
-  struct GNUNET_TIME_Absolute age;
-
-  /**
-   * When was this connection last used? (by us sending or
-   * receiving a PAYLOAD message on it)
-   */
-  struct GNUNET_TIME_Absolute last_use;
-
-  /**
-   * How many packets that ought to generate an ACK did we send via
-   * this connection?
-   */
-  unsigned long long num_acked_transmissions;
-
-  /**
-   * Number of packets that were sent via this connection did actually
-   * receive an ACK?  (Note: ACKs may be transmitted and lost via
-   * other connections, so this value should only be interpreted
-   * relative to @e num_acked_transmissions and in relation to other
-   * connections.)
-   */
-  unsigned long long num_successes;
-
-};
-
-
-/**
- * Obtain performance @a metrics from @a cc.
- *
- * @param cc connection to query
- * @return the metrics
- */
-const struct CadetConnectionMetrics *
-GCC_get_metrics (struct CadetConnection *cc);
-
-
-/**
- * Handle encrypted message.
- *
- * @param cc connection that received encrypted message
- * @param msg the encrypted message to decrypt
- */
-void
-GCC_handle_encrypted (struct CadetConnection *cc,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg);
-
-
-/**
- * We sent a message for which we expect to receive an ACK via
- * the connection identified by @a cti.
- *
- * @param cid connection identifier where we expect an ACK
- */
-void
-GCC_ack_expected (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
-
-
-/**
- * We observed an ACK for a message that was originally sent via
- * the connection identified by @a cti.
- *
- * @param cid connection identifier where we got an ACK for a message
- *            that was originally sent via this connection (the ACK
- *            may have gotten back to us via a different connection).
- */
-void
-GCC_ack_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
-
-
-/**
- * We observed some the given @a latency on the connection
- * identified by @a cti.  (The same connection was taken
- * in both directions.)
- *
- * @param cti connection identifier where we measured latency
- * @param latency the observed latency
- */
-void
-GCC_latency_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
-                      struct GNUNET_TIME_Relative latency);
-
-
-/**
- * Return the tunnel associated with this connection.
- *
- * @param cc connection to query
- * @return corresponding entry in the tunnel's connection list
- */
-struct CadetTConnection *
-GCC_get_ct (struct CadetConnection *cc);
-
-
-/**
- * Obtain the path used by this connection.
- *
- * @param cc connection
- * @return path to @a cc
- */
-struct CadetPeerPath *
-GCC_get_path (struct CadetConnection *cc);
-
-
-/**
- * Obtain unique ID for the connection.
- *
- * @param cc connection.
- * @return unique number of the connection
- */
-const struct GNUNET_CADET_ConnectionTunnelIdentifier *
-GCC_get_id (struct CadetConnection *cc);
-
-
-/**
- * Get a (static) string for a connection.
- *
- * @param cc Connection.
- */
-const char *
-GCC_2s (const struct CadetConnection *cc);
-
-
-/**
- * Log connection info.
- *
- * @param cc connection
- * @param level Debug level to use.
- */
-void
-GCC_debug (struct CadetConnection *cc,
-           enum GNUNET_ErrorType level);
-
-
-#endif
diff --git a/src/cadet/gnunet-service-cadet-new_dht.c b/src/cadet/gnunet-service-cadet-new_dht.c
deleted file mode 100644
index 849562f23..000000000
--- a/src/cadet/gnunet-service-cadet-new_dht.c
+++ /dev/null
@@ -1,351 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/gnunet-service-cadet-new_dht.c
- * @brief Information we track per peer.
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_statistics_service.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_dht.h"
-#include "gnunet-service-cadet-new_hello.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
-
-/**
- * How long do we wait before first announcing our presence to the DHT.
- * Used to wait for our HELLO to be available.  Note that we also get
- * notifications when our HELLO is ready, so this is just the maximum
- * we wait for the first notification.
- */
-#define STARTUP_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 500)
-
-/**
- * How long do we wait after we get an updated HELLO before publishing?
- * Allows for the HELLO to be updated again quickly, for example in
- * case multiple addresses changed and we got a partial update.
- */
-#define CHANGE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 100)
-
-
-#define LOG(level, ...) GNUNET_log_from (level,"cadet-dht",__VA_ARGS__)
-
-
-/**
- * Handle for DHT searches.
- */
-struct GCD_search_handle
-{
-  /**
-   * DHT_GET handle.
-   */
-  struct GNUNET_DHT_GetHandle *dhtget;
-
-};
-
-
-/**
- * Handle to use DHT.
- */
-static struct GNUNET_DHT_Handle *dht_handle;
-
-/**
- * How often to PUT own ID in the DHT.
- */
-static struct GNUNET_TIME_Relative id_announce_time;
-
-/**
- * DHT replication level, see DHT API: #GNUNET_DHT_get_start(), #GNUNET_DHT_put().
- */
-static unsigned long long dht_replication_level;
-
-/**
- * Task to periodically announce itself in the network.
- */
-static struct GNUNET_SCHEDULER_Task *announce_id_task;
-
-/**
- * Delay for the next ID announce.
- */
-static struct GNUNET_TIME_Relative announce_delay;
-
-
-/**
- * Function to process paths received for a new peer addition. The recorded
- * paths form the initial tunnel, which can be optimized later.
- * Called on each result obtained for the DHT search.
- *
- * @param cls closure
- * @param exp when will this value expire
- * @param key key of the result
- * @param get_path path of the get request
- * @param get_path_length lenght of @a get_path
- * @param put_path path of the put request
- * @param put_path_length length of the @a put_path
- * @param type type of the result
- * @param size number of bytes in data
- * @param data pointer to the result data
- */
-static void
-dht_get_id_handler (void *cls, struct GNUNET_TIME_Absolute exp,
-                    const struct GNUNET_HashCode *key,
-                    const struct GNUNET_PeerIdentity *get_path,
-                    unsigned int get_path_length,
-                    const struct GNUNET_PeerIdentity *put_path,
-                    unsigned int put_path_length,
-                    enum GNUNET_BLOCK_Type type,
-                    size_t size,
-                    const void *data)
-{
-  const struct GNUNET_HELLO_Message *hello = data;
-  struct CadetPeer *peer;
-
-  GCPP_try_path_from_dht (get_path,
-                          get_path_length,
-                          put_path,
-                          put_path_length);
-  if ( (size >= sizeof (struct GNUNET_HELLO_Message)) &&
-       (ntohs (hello->header.size) == size) &&
-       (size == GNUNET_HELLO_size (hello)) )
-  {
-    peer = GCP_get (&put_path[0],
-                    GNUNET_YES);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got HELLO for %s\n",
-         GCP_2s (peer));
-    GCP_set_hello (peer,
-                   hello);
-  }
-}
-
-
-/**
- * Periodically announce self id in the DHT
- *
- * @param cls closure
- */
-static void
-announce_id (void *cls)
-{
-  struct GNUNET_HashCode phash;
-  const struct GNUNET_HELLO_Message *hello;
-  size_t size;
-  struct GNUNET_TIME_Absolute expiration;
-  struct GNUNET_TIME_Relative next_put;
-
-  hello = GCH_get_mine ();
-  size = (NULL != hello) ? GNUNET_HELLO_size (hello) : 0;
-  if (0 == size)
-  {
-    expiration = GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get (),
-                                           announce_delay);
-    announce_delay = GNUNET_TIME_STD_BACKOFF (announce_delay);
-  }
-  else
-  {
-    expiration = GNUNET_HELLO_get_last_expiration (hello);
-    announce_delay = GNUNET_TIME_UNIT_SECONDS;
-  }
-
-  /* Call again in id_announce_time, unless HELLO expires first,
-   * but wait at least 1s. */
-  next_put
-    = GNUNET_TIME_absolute_get_remaining (expiration);
-  next_put
-    = GNUNET_TIME_relative_min (next_put,
-                                id_announce_time);
-  next_put
-    = GNUNET_TIME_relative_max (next_put,
-                                GNUNET_TIME_UNIT_SECONDS);
-  announce_id_task
-    = GNUNET_SCHEDULER_add_delayed (next_put,
-                                    &announce_id,
-                                    cls);
-  GNUNET_STATISTICS_update (stats,
-                            "# DHT announce",
-                            1,
-                            GNUNET_NO);
-  memset (&phash,
-          0,
-          sizeof (phash));
-  GNUNET_memcpy (&phash,
-                 &my_full_id,
-                 sizeof (my_full_id));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Announcing my HELLO (%u bytes) in the DHT\n",
-       size);
-  GNUNET_DHT_put (dht_handle,   /* DHT handle */
-                  &phash,       /* Key to use */
-                  dht_replication_level,     /* Replication level */
-                  GNUNET_DHT_RO_RECORD_ROUTE
-                  | GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,    /* DHT options */
-                  GNUNET_BLOCK_TYPE_DHT_HELLO,       /* Block type */
-                  size,  /* Size of the data */
-                  (const char *) hello, /* Data itself */
-                  expiration,  /* Data expiration */
-                  NULL,         /* Continuation */
-                  NULL);        /* Continuation closure */
-}
-
-
-/**
- * Function called by the HELLO subsystem whenever OUR hello
- * changes. Re-triggers the DHT PUT immediately.
- */
-void
-GCD_hello_update ()
-{
-  if (NULL == announce_id_task)
-    return; /* too early */
-  GNUNET_SCHEDULER_cancel (announce_id_task);
-  announce_id_task
-    = GNUNET_SCHEDULER_add_delayed (CHANGE_DELAY,
-                                    &announce_id,
-                                    NULL);
-}
-
-
-/**
- * Initialize the DHT subsystem.
- *
- * @param c Configuration.
- */
-void
-GCD_init (const struct GNUNET_CONFIGURATION_Handle *c)
-{
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c,
-                                             "CADET",
-                                             "DHT_REPLICATION_LEVEL",
-                                             &dht_replication_level))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET",
-                               "DHT_REPLICATION_LEVEL",
-                               "USING DEFAULT");
-    dht_replication_level = 3;
-  }
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c,
-                                           "CADET",
-                                           "ID_ANNOUNCE_TIME",
-                                           &id_announce_time))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
-                               "CADET",
-                               "ID_ANNOUNCE_TIME",
-                               "MISSING");
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-
-  dht_handle = GNUNET_DHT_connect (c,
-                                   64);
-  GNUNET_break (NULL != dht_handle);
-  announce_delay = GNUNET_TIME_UNIT_SECONDS;
-  announce_id_task = GNUNET_SCHEDULER_add_delayed (STARTUP_DELAY,
-                                                   &announce_id,
-                                                   NULL);
-}
-
-
-/**
- * Shut down the DHT subsystem.
- */
-void
-GCD_shutdown (void)
-{
-  if (NULL != dht_handle)
-  {
-    GNUNET_DHT_disconnect (dht_handle);
-    dht_handle = NULL;
-  }
-  if (NULL != announce_id_task)
-  {
-    GNUNET_SCHEDULER_cancel (announce_id_task);
-    announce_id_task = NULL;
-  }
-}
-
-
-/**
- * Search DHT for paths to @a peeR_id
- *
- * @param peer_id peer to search for
- * @return handle to abort search
- */
-struct GCD_search_handle *
-GCD_search (const struct GNUNET_PeerIdentity *peer_id)
-{
-  struct GNUNET_HashCode phash;
-  struct GCD_search_handle *h;
-
-  GNUNET_STATISTICS_update (stats,
-                            "# DHT search",
-                            1,
-                            GNUNET_NO);
-  memset (&phash,
-          0,
-          sizeof (phash));
-  GNUNET_memcpy (&phash,
-                 peer_id,
-                 sizeof (*peer_id));
-
-  h = GNUNET_new (struct GCD_search_handle);
-  h->dhtget = GNUNET_DHT_get_start (dht_handle,    /* handle */
-                                    GNUNET_BLOCK_TYPE_DHT_HELLO, /* type */
-                                    &phash,     /* key to search */
-                                    dht_replication_level, /* replication level */
-                                    GNUNET_DHT_RO_RECORD_ROUTE |
-                                    GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
-                                    NULL,       /* xquery */
-                                    0,     /* xquery bits */
-                                    &dht_get_id_handler,
-                                    h);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Starting DHT GET for peer %s (%p)\n",
-       GNUNET_i2s (peer_id),
-       h);
-  return h;
-}
-
-
-/**
- * Stop DHT search started with #GCD_search().
- *
- * @param h handle to search to stop
- */
-void
-GCD_search_stop (struct GCD_search_handle *h)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Stopping DHT GET %p\n",
-       h);
-  GNUNET_DHT_get_stop (h->dhtget);
-  GNUNET_free (h);
-}
-
-/* end of gnunet-service-cadet_dht.c */
diff --git a/src/cadet/gnunet-service-cadet-new_dht.h b/src/cadet/gnunet-service-cadet-new_dht.h
deleted file mode 100644
index 5d7ab29a0..000000000
--- a/src/cadet/gnunet-service-cadet-new_dht.h
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet_dht.h
- * @brief cadet service; dealing with DHT requests and results
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * All functions in this file should use the prefix GCD (Gnunet Cadet Dht)
- */
-#ifndef GNUNET_SERVICE_CADET_DHT_H
-#define GNUNET_SERVICE_CADET_DHT_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-/**
- * Handle for DHT search operation.
- */
-struct GCD_search_handle;
-
-
-/**
- * Initialize the DHT subsystem.
- *
- * @param c Configuration.
- */
-void
-GCD_init (const struct GNUNET_CONFIGURATION_Handle *c);
-
-
-/**
- * Shut down the DHT subsystem.
- */
-void
-GCD_shutdown (void);
-
-
-/**
- * Function called by the HELLO subsystem whenever OUR hello
- * changes. Re-triggers the DHT PUT immediately.
- */
-void
-GCD_hello_update (void);
-
-/**
- * Search DHT for paths to @a peeR_id
- *
- * @param peer_id peer to search for
- * @return handle to abort search
- */
-struct GCD_search_handle *
-GCD_search (const struct GNUNET_PeerIdentity *peer_id);
-
-
-/**
- * Stop DHT search started with #GCD_search().
- *
- * @param h handle to search to stop
- */
-void
-GCD_search_stop (struct GCD_search_handle *h);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_CADET_SERVICE_DHT_H */
-#endif
-/* end of gnunet-service-cadet_dht.h */
diff --git a/src/cadet/gnunet-service-cadet-new_hello.c b/src/cadet/gnunet-service-cadet-new_hello.c
deleted file mode 100644
index a24325ada..000000000
--- a/src/cadet/gnunet-service-cadet-new_hello.c
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2014, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/gnunet-service-cadet-new_hello.c
- * @brief spread knowledge about how to contact other peers from PEERINFO
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * TODO:
- * - is most of this necessary/helpful?
- * - should we not simply restrict this to OUR hello?
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#include "gnunet_statistics_service.h"
-#include "gnunet_peerinfo_service.h"
-#include "cadet_protocol.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_dht.h"
-#include "gnunet-service-cadet-new_hello.h"
-#include "gnunet-service-cadet-new_peer.h"
-
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-hll",__VA_ARGS__)
-
-/**
- * Hello message of local peer.
- */
-static struct GNUNET_HELLO_Message *mine;
-
-/**
- * Handle to peerinfo service.
- */
-static struct GNUNET_PEERINFO_Handle *peerinfo;
-
-/**
- * Iterator context.
- */
-static struct GNUNET_PEERINFO_NotifyContext *nc;
-
-
-/**
- * Process each hello message received from peerinfo.
- *
- * @param cls Closure (unused).
- * @param peer Identity of the peer.
- * @param hello Hello of the peer.
- * @param err_msg Error message.
- */
-static void
-got_hello (void *cls,
-           const struct GNUNET_PeerIdentity *id,
-           const struct GNUNET_HELLO_Message *hello,
-           const char *err_msg)
-{
-  struct CadetPeer *peer;
-
-  if ( (NULL == id) ||
-       (NULL == hello) )
-    return;
-  if (0 == memcmp (id,
-                   &my_full_id,
-                   sizeof (struct GNUNET_PeerIdentity)))
-  {
-    GNUNET_free_non_null (mine);
-    mine = (struct GNUNET_HELLO_Message *) GNUNET_copy_message (&hello->header);
-    GCD_hello_update ();
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Hello for %s (%d bytes), expires on %s\n",
-       GNUNET_i2s (id),
-       GNUNET_HELLO_size (hello),
-       GNUNET_STRINGS_absolute_time_to_string (GNUNET_HELLO_get_last_expiration (hello)));
-  peer = GCP_get (id,
-                  GNUNET_YES);
-  GCP_set_hello (peer,
-                 hello);
-}
-
-
-/**
- * Initialize the hello subsystem.
- *
- * @param c Configuration.
- */
-void
-GCH_init (const struct GNUNET_CONFIGURATION_Handle *c)
-{
-  GNUNET_assert (NULL == nc);
-  peerinfo = GNUNET_PEERINFO_connect (c);
-  nc = GNUNET_PEERINFO_notify (c,
-                               GNUNET_NO,
-                               &got_hello,
-                               NULL);
-}
-
-
-/**
- * Shut down the hello subsystem.
- */
-void
-GCH_shutdown ()
-{
-  if (NULL != nc)
-  {
-    GNUNET_PEERINFO_notify_cancel (nc);
-    nc = NULL;
-  }
-  if (NULL != peerinfo)
-  {
-    GNUNET_PEERINFO_disconnect (peerinfo);
-    peerinfo = NULL;
-  }
-  if (NULL != mine)
-  {
-    GNUNET_free (mine);
-    mine = NULL;
-  }
-}
-
-
-/**
- * Get own hello message.
- *
- * @return Own hello message.
- */
-const struct GNUNET_HELLO_Message *
-GCH_get_mine (void)
-{
-  return mine;
-}
-
-/* end of gnunet-service-cadet-new_hello.c */
diff --git a/src/cadet/gnunet-service-cadet-new_hello.h b/src/cadet/gnunet-service-cadet-new_hello.h
deleted file mode 100644
index 4291ae985..000000000
--- a/src/cadet/gnunet-service-cadet-new_hello.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2014, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet_hello.h
- * @brief cadet service; dealing with hello messages
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * All functions in this file should use the prefix GCH (Gnunet Cadet Hello)
- */
-
-#ifndef GNUNET_SERVICE_CADET_HELLO_H
-#define GNUNET_SERVICE_CADET_HELLO_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_hello_lib.h"
-
-
-/**
- * Initialize the hello subsystem.
- *
- * @param c Configuration.
- */
-void
-GCH_init (const struct GNUNET_CONFIGURATION_Handle *c);
-
-
-/**
- * Shut down the hello subsystem.
- */
-void
-GCH_shutdown (void);
-
-
-/**
- * Get own hello message.
- *
- * @return Own hello message.
- */
-const struct GNUNET_HELLO_Message *
-GCH_get_mine (void);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_CADET_SERVICE_HELLO_H */
-#endif
-/* end of gnunet-cadet-service_hello.h */
diff --git a/src/cadet/gnunet-service-cadet-new_peer.c b/src/cadet/gnunet-service-cadet-new_peer.c
deleted file mode 100644
index 350c8efae..000000000
--- a/src/cadet/gnunet-service-cadet-new_peer.c
+++ /dev/null
@@ -1,1477 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new_peer.c
- * @brief Information we track per peer.
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- *
- * TODO:
- * - optimize stopping/restarting DHT search to situations
- *   where we actually need it (i.e. not if we have a direct connection,
- *   or if we already have plenty of good short ones, or maybe even
- *   to take a break if we have some connections and have searched a lot (?))
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_hello_lib.h"
-#include "gnunet_signatures.h"
-#include "gnunet_transport_service.h"
-#include "gnunet_ats_service.h"
-#include "gnunet_core_service.h"
-#include "gnunet_statistics_service.h"
-#include "cadet_protocol.h"
-#include "gnunet-service-cadet-new.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_dht.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-
-
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-per",__VA_ARGS__)
-
-
-/**
- * How long do we wait until tearing down an idle peer?
- */
-#define IDLE_PEER_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
-
-/**
- * How long do we keep paths around if we no longer care about the peer?
- */
-#define IDLE_PATH_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 2)
-
-
-
-
-/**
- * Data structure used to track whom we have to notify about changes
- * to our message queue.
- */
-struct GCP_MessageQueueManager
-{
-
-  /**
-   * Kept in a DLL.
-   */
-  struct GCP_MessageQueueManager *next;
-
-  /**
-   * Kept in a DLL.
-   */
-  struct GCP_MessageQueueManager *prev;
-
-  /**
-   * Function to call with updated message queue object.
-   */
-  GCP_MessageQueueNotificationCallback cb;
-
-  /**
-   * Closure for @e cb.
-   */
-  void *cb_cls;
-
-  /**
-   * The peer this is for.
-   */
-  struct CadetPeer *cp;
-
-  /**
-   * Envelope this manager would like to transmit once it is its turn.
-   */
-  struct GNUNET_MQ_Envelope *env;
-
-};
-
-
-/**
- * Struct containing all information regarding a given peer
- */
-struct CadetPeer
-{
-  /**
-   * ID of the peer
-   */
-  struct GNUNET_PeerIdentity pid;
-
-  /**
-   * Last time we heard from this peer (currently not used!)
-   */
-  struct GNUNET_TIME_Absolute last_contactXXX;
-
-  /**
-   * Array of DLLs of paths traversing the peer, organized by the
-   * offset of the peer on the larger path.
-   */
-  struct CadetPeerPathEntry **path_heads;
-
-  /**
-   * Array of DLL of paths traversing the peer, organized by the
-   * offset of the peer on the larger path.
-   */
-  struct CadetPeerPathEntry **path_tails;
-
-  /**
-   * Notifications to call when @e core_mq changes.
-   */
-  struct GCP_MessageQueueManager *mqm_head;
-
-  /**
-   * Notifications to call when @e core_mq changes.
-   */
-  struct GCP_MessageQueueManager *mqm_tail;
-
-  /**
-   * Pointer to first "ready" entry in @e mqm_head.
-   */
-  struct GCP_MessageQueueManager *mqm_ready_ptr;
-
-  /**
-   * MIN-heap of paths owned by this peer (they also end at this
-   * peer).  Ordered by desirability.
-   */
-  struct GNUNET_CONTAINER_Heap *path_heap;
-
-  /**
-   * Handle to stop the DHT search for paths to this peer
-   */
-  struct GCD_search_handle *search_h;
-
-  /**
-   * Task to clean up @e path_heap asynchronously.
-   */
-  struct GNUNET_SCHEDULER_Task *heap_cleanup_task;
-
-  /**
-   * Task to destroy this entry.
-   */
-  struct GNUNET_SCHEDULER_Task *destroy_task;
-
-  /**
-   * Tunnel to this peer, if any.
-   */
-  struct CadetTunnel *t;
-
-  /**
-   * Connections that go through this peer; indexed by tid.
-   */
-  struct GNUNET_CONTAINER_MultiShortmap *connections;
-
-  /**
-   * Handle for core transmissions.
-   */
-  struct GNUNET_MQ_Handle *core_mq;
-
-  /**
-   * Hello message of the peer.
-   */
-  struct GNUNET_HELLO_Message *hello;
-
-  /**
-   * Handle to us offering the HELLO to the transport.
-   */
-  struct GNUNET_TRANSPORT_OfferHelloHandle *hello_offer;
-
-  /**
-   * Handle to our ATS request asking ATS to suggest an address
-   * to TRANSPORT for this peer (to establish a direct link).
-   */
-  struct GNUNET_ATS_ConnectivitySuggestHandle *connectivity_suggestion;
-
-  /**
-   * How many messages are in the queue to this peer.
-   */
-  unsigned int queue_n;
-
-  /**
-   * How many paths do we have to this peer (in all @e path_heads DLLs combined).
-   */
-  unsigned int num_paths;
-
-  /**
-   * Sum over all of the offsets of all of the paths in the @a path_heads DLLs.
-   * Used to speed-up @GCP_get_desirability_of_path() calculation.
-   */
-  unsigned int off_sum;
-
-  /**
-   * Number of message queue managers of this peer that have a message in waiting.
-   *
-   * Used to quickly see if we need to bother scanning the @e msm_head DLL.
-   * TODO: could be replaced by another DLL that would then allow us to avoid
-   * the O(n)-scan of the DLL for ready entries!
-   */
-  unsigned int mqm_ready_counter;
-
-  /**
-   * Current length of the @e path_heads and @path_tails arrays.
-   * The arrays should be grown as needed.
-   */
-  unsigned int path_dll_length;
-
-};
-
-
-/**
- * Get the static string for a peer ID.
- *
- * @param cp Peer.
- * @return Static string for it's ID.
- */
-const char *
-GCP_2s (const struct CadetPeer *cp)
-{
-  static char buf[32];
-
-  GNUNET_snprintf (buf,
-                   sizeof (buf),
-                   "P(%s)",
-                   GNUNET_i2s (&cp->pid));
-  return buf;
-}
-
-
-/**
- * Calculate how desirable a path is for @a cp if @a cp
- * is at offset @a off.
- *
- * The 'desirability_table.c' program can be used to compute a list of
- * sample outputs for different scenarios.  Basically, we score paths
- * lower if there are many alternatives, and higher if they are
- * shorter than average, and very high if they are much shorter than
- * average and without many alternatives.
- *
- * @param cp a peer reachable via a path
- * @param off offset of @a cp in the path
- * @return score how useful a path is to reach @a cp,
- *         positive scores mean path is more desirable
- */
-double
-GCP_get_desirability_of_path (struct CadetPeer *cp,
-                              unsigned int off)
-{
-  unsigned int num_alts = cp->num_paths;
-  unsigned int off_sum;
-  double avg_sum;
-  double path_delta;
-  double weight_alts;
-
-  GNUNET_assert (num_alts >= 1); /* 'path' should be in there! */
-  GNUNET_assert (0 != cp->path_dll_length);
-
-  /* We maintain 'off_sum' in 'peer' and thereby
-     avoid the SLOW recalculation each time. Kept here
-     just to document what is going on. */
-#if SLOW
-  off_sum = 0;
-  for (unsigned int j=0;j<cp->path_dll_length;j++)
-    for (struct CadetPeerPathEntry *pe = cp->path_heads[j];
-         NULL != pe;
-         pe = pe->next)
-      off_sum += j;
-  GNUNET_assert (off_sum == cp->off_sum);
-#else
-  off_sum = cp->off_sum;
-#endif
-  avg_sum = off_sum * 1.0 / cp->path_dll_length;
-  path_delta = off - avg_sum;
-  /* path_delta positiv: path off of peer above average (bad path for peer),
-     path_delta negativ: path off of peer below average (good path for peer) */
-  if (path_delta <= - 1.0)
-    weight_alts = - num_alts / path_delta; /* discount alternative paths */
-  else if (path_delta >= 1.0)
-    weight_alts = num_alts * path_delta; /* overcount alternative paths */
-  else
-    weight_alts = num_alts; /* count alternative paths normally */
-
-
-  /* off+1: long paths are generally harder to find and thus count
-     a bit more as they get longer.  However, above-average paths
-     still need to count less, hence the squaring of that factor. */
-  return (off + 1.0) / (weight_alts * weight_alts);
-}
-
-
-/**
- * This peer is no longer be needed, clean it up now.
- *
- * @param cls peer to clean up
- */
-static void
-destroy_peer (void *cls)
-{
-  struct CadetPeer *cp = cls;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying state about peer %s\n",
-       GCP_2s (cp));
-  cp->destroy_task = NULL;
-  GNUNET_assert (NULL == cp->t);
-  GNUNET_assert (NULL == cp->core_mq);
-  GNUNET_assert (0 == cp->num_paths);
-  for (unsigned int i=0;i<cp->path_dll_length;i++)
-    GNUNET_assert (NULL == cp->path_heads[i]);
-  GNUNET_assert (0 == GNUNET_CONTAINER_multishortmap_size (cp->connections));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multipeermap_remove (peers,
-                                                       &cp->pid,
-                                                       cp));
-  GNUNET_free_non_null (cp->path_heads);
-  GNUNET_free_non_null (cp->path_tails);
-  cp->path_dll_length = 0;
-  if (NULL != cp->search_h)
-  {
-    GCD_search_stop (cp->search_h);
-    cp->search_h = NULL;
-  }
-  /* FIXME: clean up search_delayedXXX! */
-
-  if (NULL != cp->hello_offer)
-  {
-    GNUNET_TRANSPORT_offer_hello_cancel (cp->hello_offer);
-    cp->hello_offer = NULL;
-  }
-  if (NULL != cp->connectivity_suggestion)
-  {
-    GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
-    cp->connectivity_suggestion = NULL;
-  }
-  GNUNET_CONTAINER_multishortmap_destroy (cp->connections);
-  if (NULL != cp->path_heap)
-  {
-    GNUNET_CONTAINER_heap_destroy (cp->path_heap);
-    cp->path_heap = NULL;
-  }
-  if (NULL != cp->heap_cleanup_task)
-  {
-    GNUNET_SCHEDULER_cancel (cp->heap_cleanup_task);
-    cp->heap_cleanup_task = NULL;
-  }
-  GNUNET_free_non_null (cp->hello);
-  /* Peer should not be freed if paths exist; if there are no paths,
-     there ought to be no connections, and without connections, no
-     notifications. Thus we can assert that mqm_head is empty at this
-     point. */
-  GNUNET_assert (NULL == cp->mqm_head);
-  GNUNET_assert (NULL == cp->mqm_ready_ptr);
-  GNUNET_free (cp);
-}
-
-
-/**
- * This peer is now on more "active" duty, activate processes related to it.
- *
- * @param cp the more-active peer
- */
-static void
-consider_peer_activate (struct CadetPeer *cp)
-{
-  uint32_t strength;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Updating peer %s activation state (%u connections)%s%s\n",
-       GCP_2s (cp),
-       GNUNET_CONTAINER_multishortmap_size (cp->connections),
-       (NULL == cp->t) ? "" : " with tunnel",
-       (NULL == cp->core_mq) ? "" : " with CORE link");
-  if (NULL != cp->destroy_task)
-  {
-    /* It's active, do not destory! */
-    GNUNET_SCHEDULER_cancel (cp->destroy_task);
-    cp->destroy_task = NULL;
-  }
-  if ( (0 == GNUNET_CONTAINER_multishortmap_size (cp->connections)) &&
-       (NULL == cp->t) )
-  {
-    /* We're just on a path or directly connected; don't bother too much */
-    if (NULL != cp->connectivity_suggestion)
-    {
-      GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
-      cp->connectivity_suggestion = NULL;
-    }
-    if (NULL != cp->search_h)
-    {
-      GCD_search_stop (cp->search_h);
-      cp->search_h = NULL;
-    }
-    return;
-  }
-  if (NULL == cp->core_mq)
-  {
-    /* Lacks direct connection, try to create one by querying the DHT */
-    if ( (NULL == cp->search_h) &&
-         (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) )
-      cp->search_h
-        = GCD_search (&cp->pid);
-  }
-  else
-  {
-    /* Have direct connection, stop DHT search if active */
-    if (NULL != cp->search_h)
-    {
-      GCD_search_stop (cp->search_h);
-      cp->search_h = NULL;
-    }
-  }
-
-  /* If we have a tunnel, our urge for connections is much bigger */
-  strength = (NULL != cp->t) ? 32 : 1;
-  if (NULL != cp->connectivity_suggestion)
-    GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
-  cp->connectivity_suggestion
-    = GNUNET_ATS_connectivity_suggest (ats_ch,
-                                       &cp->pid,
-                                       strength);
-}
-
-
-/**
- * This peer may no longer be needed, consider cleaning it up.
- *
- * @param cp peer to clean up
- */
-static void
-consider_peer_destroy (struct CadetPeer *cp);
-
-
-/**
- * We really no longere care about a peer, stop hogging memory with paths to it.
- * Afterwards, see if there is more to be cleaned up about this peer.
- *
- * @param cls a `struct CadetPeer`.
- */
-static void
-drop_paths (void *cls)
-{
-  struct CadetPeer *cp = cls;
-  struct CadetPeerPath *path;
-
-  cp->destroy_task = NULL;
-  while (NULL != (path = GNUNET_CONTAINER_heap_remove_root (cp->path_heap)))
-    GCPP_release (path);
-  consider_peer_destroy (cp);
-}
-
-
-/**
- * This peer may no longer be needed, consider cleaning it up.
- *
- * @param cp peer to clean up
- */
-static void
-consider_peer_destroy (struct CadetPeer *cp)
-{
-  struct GNUNET_TIME_Relative exp;
-
-  if (NULL != cp->destroy_task)
-  {
-    GNUNET_SCHEDULER_cancel (cp->destroy_task);
-    cp->destroy_task = NULL;
-  }
-  if (NULL != cp->t)
-    return; /* still relevant! */
-  if (NULL != cp->core_mq)
-    return; /* still relevant! */
-  if (0 != GNUNET_CONTAINER_multishortmap_size (cp->connections))
-    return; /* still relevant! */
-  if ( (NULL != cp->path_heap) &&
-       (0 < GNUNET_CONTAINER_heap_get_size (cp->path_heap)) )
-  {
-    cp->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_PATH_TIMEOUT,
-                                                     &drop_paths,
-                                                     cp);
-    return;
-  }
-  if (0 != cp->num_paths)
-    return; /* still relevant! */
-  if (NULL != cp->hello)
-  {
-    /* relevant only until HELLO expires */
-    exp = GNUNET_TIME_absolute_get_remaining (GNUNET_HELLO_get_last_expiration (cp->hello));
-    cp->destroy_task = GNUNET_SCHEDULER_add_delayed (exp,
-                                                     &destroy_peer,
-                                                     cp);
-    return;
-  }
-  cp->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_PEER_TIMEOUT,
-                                                   &destroy_peer,
-                                                   cp);
-}
-
-
-/**
- * Set the message queue to @a mq for peer @a cp and notify watchers.
- *
- * @param cp peer to modify
- * @param mq message queue to set (can be NULL)
- */
-void
-GCP_set_mq (struct CadetPeer *cp,
-            struct GNUNET_MQ_Handle *mq)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Message queue for peer %s is now %p\n",
-       GCP_2s (cp),
-       mq);
-  cp->core_mq = mq;
-  for (struct GCP_MessageQueueManager *mqm = cp->mqm_head, *next;
-       NULL != mqm;
-       mqm = next)
-  {
-    /* Save next pointer in case mqm gets freed by the callback */
-    next = mqm->next;
-    if (NULL == mq)
-    {
-      if (NULL != mqm->env)
-      {
-        GNUNET_MQ_discard (mqm->env);
-        mqm->env = NULL;
-        mqm->cb (mqm->cb_cls,
-                 GNUNET_SYSERR);
-      }
-      else
-      {
-        mqm->cb (mqm->cb_cls,
-                 GNUNET_NO);
-      }
-    }
-    else
-    {
-      GNUNET_assert (NULL == mqm->env);
-      mqm->cb (mqm->cb_cls,
-               GNUNET_YES);
-    }
-  }
-  if ( (NULL != mq) ||
-       (NULL != cp->t) )
-    consider_peer_activate (cp);
-  else
-    consider_peer_destroy (cp);
-
-  if ( (NULL != mq) &&
-       (NULL != cp->t) )
-  {
-    /* have a new, direct path to the target, notify tunnel */
-    struct CadetPeerPath *path;
-
-    path = GCPP_get_path_from_route (1,
-                                     &cp->pid);
-    GCT_consider_path (cp->t,
-                       path,
-                       0);
-  }
-}
-
-
-/**
- * Debug function should NEVER return true in production code, useful to
- * simulate losses for testcases.
- *
- * @return #GNUNET_YES or #GNUNET_NO with the decision to drop.
- */
-static int
-should_I_drop (void)
-{
-  if (0 == drop_percent)
-    return GNUNET_NO;
-  if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                101) < drop_percent)
-    return GNUNET_YES;
-  return GNUNET_NO;
-}
-
-
-/**
- * Function called when CORE took one of the messages from
- * a message queue manager and transmitted it.
- *
- * @param cls the `struct CadetPeeer` where we made progress
- */
-static void
-mqm_send_done (void *cls);
-
-
-/**
- * Transmit current envelope from this @a mqm.
- *
- * @param mqm mqm to transmit message for now
- */
-static void
-mqm_execute (struct GCP_MessageQueueManager *mqm)
-{
-  struct CadetPeer *cp = mqm->cp;
-
-  /* Move ready pointer to the next entry that might be ready. */
-  if ( (mqm == cp->mqm_ready_ptr) &&
-       (NULL != mqm->next) )
-    cp->mqm_ready_ptr = mqm->next;
-  /* Move entry to the end of the DLL, to be fair. */
-  if (mqm != cp->mqm_tail)
-  {
-    GNUNET_CONTAINER_DLL_remove (cp->mqm_head,
-                                 cp->mqm_tail,
-                                 mqm);
-    GNUNET_CONTAINER_DLL_insert_tail (cp->mqm_head,
-                                      cp->mqm_tail,
-                                      mqm);
-  }
-  cp->mqm_ready_counter--;
-  if (GNUNET_YES == should_I_drop ())
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "DROPPING message to peer %s from MQM %p\n",
-         GCP_2s (cp),
-         mqm);
-    GNUNET_MQ_discard (mqm->env);
-    mqm->env = NULL;
-    mqm_send_done (cp);
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Sending to peer %s from MQM %p\n",
-         GCP_2s (cp),
-         mqm);
-    GNUNET_MQ_send (cp->core_mq,
-                    mqm->env);
-    mqm->env = NULL;
-  }
-  mqm->cb (mqm->cb_cls,
-           GNUNET_YES);
-}
-
-
-/**
- * Find the next ready message in the queue (starting
- * the search from the `cp->mqm_ready_ptr`) and if possible
- * execute the transmission.
- *
- * @param cp peer to try to send the next ready message to
- */
-static void
-send_next_ready (struct CadetPeer *cp)
-{
-  struct GCP_MessageQueueManager *mqm;
-
-  if (0 == cp->mqm_ready_counter)
-    return;
-  while ( (NULL != (mqm = cp->mqm_ready_ptr)) &&
-          (NULL == mqm->env) )
-    cp->mqm_ready_ptr = mqm->next;
-  if (NULL == mqm)
-    return; /* nothing to do */
-  mqm_execute (mqm);
-}
-
-
-/**
- * Function called when CORE took one of the messages from
- * a message queue manager and transmitted it.
- *
- * @param cls the `struct CadetPeeer` where we made progress
- */
-static void
-mqm_send_done (void *cls)
-{
-  struct CadetPeer *cp = cls;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending to peer %s completed\n",
-       GCP_2s (cp));
-  send_next_ready (cp);
-}
-
-
-/**
- * Send the message in @a env to @a cp.
- *
- * @param mqm the message queue manager to use for transmission
- * @param env envelope with the message to send; must NOT
- *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
- */
-void
-GCP_send (struct GCP_MessageQueueManager *mqm,
-          struct GNUNET_MQ_Envelope *env)
-{
-  struct CadetPeer *cp = mqm->cp;
-
-  GNUNET_assert (NULL != env);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Queueing message to peer %s in MQM %p\n",
-       GCP_2s (cp),
-       mqm);
-  GNUNET_assert (NULL != cp->core_mq);
-  GNUNET_assert (NULL == mqm->env);
-  GNUNET_MQ_notify_sent (env,
-                         &mqm_send_done,
-                         cp);
-  mqm->env = env;
-  cp->mqm_ready_counter++;
-  if (mqm != cp->mqm_ready_ptr)
-    cp->mqm_ready_ptr = cp->mqm_head;
-  if (1 == cp->mqm_ready_counter)
-    cp->mqm_ready_ptr = mqm;
-  if (0 != GNUNET_MQ_get_length (cp->core_mq))
-    return;
-  send_next_ready (cp);
-}
-
-
-/**
- * Function called to destroy a peer now.
- *
- * @param cls NULL
- * @param pid identity of the peer (unused)
- * @param value the `struct CadetPeer` to clean up
- * @return #GNUNET_OK (continue to iterate)
- */
-static int
-destroy_iterator_cb (void *cls,
-                     const struct GNUNET_PeerIdentity *pid,
-                     void *value)
-{
-  struct CadetPeer *cp = value;
-
-  if (NULL != cp->destroy_task)
-  {
-    GNUNET_SCHEDULER_cancel (cp->destroy_task);
-    cp->destroy_task = NULL;
-  }
-  destroy_peer (cp);
-  return GNUNET_OK;
-}
-
-
-/**
- * Clean up all entries about all peers.
- * Must only be called after all tunnels, CORE-connections and
- * connections are down.
- */
-void
-GCP_destroy_all_peers ()
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying all peers now\n");
-  GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                         &destroy_iterator_cb,
-                                         NULL);
-}
-
-
-/**
- * Drop all paths owned by this peer, and do not
- * allow new ones to be added: We are shutting down.
- *
- * @param cp peer to drop paths to
- */
-void
-GCP_drop_owned_paths (struct CadetPeer *cp)
-{
-  struct CadetPeerPath *path;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying all paths to %s\n",
-       GCP_2s (cp));
-  while (NULL != (path =
-                  GNUNET_CONTAINER_heap_remove_root (cp->path_heap)))
-    GCPP_release (path);
-  GNUNET_CONTAINER_heap_destroy (cp->path_heap);
-  cp->path_heap = NULL;
-}
-
-
-/**
- * Add an entry to the DLL of all of the paths that this peer is on.
- *
- * @param cp peer to modify
- * @param entry an entry on a path
- * @param off offset of this peer on the path
- */
-void
-GCP_path_entry_add (struct CadetPeer *cp,
-                    struct CadetPeerPathEntry *entry,
-                    unsigned int off)
-{
-  GNUNET_assert (cp == GCPP_get_peer_at_offset (entry->path,
-                                                off));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Discovered that peer %s is on path %s at offset %u\n",
-       GCP_2s (cp),
-       GCPP_2s (entry->path),
-       off);
-  if (off >= cp->path_dll_length)
-  {
-    unsigned int len = cp->path_dll_length;
-
-    GNUNET_array_grow (cp->path_heads,
-                       len,
-                       off + 4);
-    GNUNET_array_grow (cp->path_tails,
-                       cp->path_dll_length,
-                       off + 4);
-  }
-  GNUNET_CONTAINER_DLL_insert (cp->path_heads[off],
-                               cp->path_tails[off],
-                               entry);
-  cp->off_sum += off;
-  cp->num_paths++;
-
-  /* If we have a tunnel to this peer, tell the tunnel that there is a
-     new path available. */
-  if (NULL != cp->t)
-    GCT_consider_path (cp->t,
-                       entry->path,
-                       off);
-
-  if ( (NULL != cp->search_h) &&
-       (DESIRED_CONNECTIONS_PER_TUNNEL <= cp->num_paths) )
-  {
-    /* Now I have enough paths, stop search */
-    GCD_search_stop (cp->search_h);
-    cp->search_h = NULL;
-  }
-  if (NULL != cp->destroy_task)
-  {
-    /* paths changed, this resets the destroy timeout counter
-       and aborts a destroy task that may no longer be valid
-       to have (as we now have more paths via this peer). */
-    consider_peer_destroy (cp);
-  }
-}
-
-
-/**
- * Remove an entry from the DLL of all of the paths that this peer is on.
- *
- * @param cp peer to modify
- * @param entry an entry on a path
- * @param off offset of this peer on the path
- */
-void
-GCP_path_entry_remove (struct CadetPeer *cp,
-                       struct CadetPeerPathEntry *entry,
-                       unsigned int off)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Removing knowledge about peer %s beging on path %s at offset %u\n",
-       GCP_2s (cp),
-       GCPP_2s (entry->path),
-       off);
-  GNUNET_CONTAINER_DLL_remove (cp->path_heads[off],
-                               cp->path_tails[off],
-                               entry);
-  GNUNET_assert (0 < cp->num_paths);
-  cp->off_sum -= off;
-  cp->num_paths--;
-  if ( (NULL == cp->core_mq) &&
-       (NULL != cp->t) &&
-       (NULL == cp->search_h) &&
-       (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) )
-    cp->search_h
-      = GCD_search (&cp->pid);
-  if (NULL == cp->destroy_task)
-  {
-    /* paths changed, we might now be ready for destruction, check again */
-    consider_peer_destroy (cp);
-  }
-}
-
-
-/**
- * Prune down the number of paths to this peer, we seem to
- * have way too many.
- *
- * @param cls the `struct CadetPeer` to maintain the path heap for
- */
-static void
-path_heap_cleanup (void *cls)
-{
-  struct CadetPeer *cp = cls;
-  struct CadetPeerPath *root;
-
-  cp->heap_cleanup_task = NULL;
-  while (GNUNET_CONTAINER_heap_get_size (cp->path_heap) >=
-         2 * DESIRED_CONNECTIONS_PER_TUNNEL)
-  {
-    /* Now we have way too many, drop least desirable UNLESS it is in use!
-       (Note that this intentionally keeps highly desireable, but currently
-       unused paths around in the hope that we might be able to switch, even
-       if the number of paths exceeds the threshold.) */
-    root = GNUNET_CONTAINER_heap_peek (cp->path_heap);
-    if (NULL !=
-        GCPP_get_connection (root,
-                             cp,
-                             GCPP_get_length (root) - 1))
-      break; /* can't fix */
-    /* Got plenty of paths to this destination, and this is a low-quality
-       one that we don't care about. Allow it to die. */
-    GNUNET_assert (root ==
-                   GNUNET_CONTAINER_heap_remove_root (cp->path_heap));
-    GCPP_release (root);
-  }
-}
-
-
-/**
- * Try adding a @a path to this @a peer.  If the peer already
- * has plenty of paths, return NULL.
- *
- * @param cp peer to which the @a path leads to
- * @param path a path looking for an owner; may not be fully initialized yet!
- * @param off offset of @a cp in @a path
- * @param force force attaching the path
- * @return NULL if this peer does not care to become a new owner,
- *         otherwise the node in the peer's path heap for the @a path.
- */
-struct GNUNET_CONTAINER_HeapNode *
-GCP_attach_path (struct CadetPeer *cp,
-                 struct CadetPeerPath *path,
-                 unsigned int off,
-                 int force)
-{
-  GNUNET_CONTAINER_HeapCostType desirability;
-  struct CadetPeerPath *root;
-  GNUNET_CONTAINER_HeapCostType root_desirability;
-  struct GNUNET_CONTAINER_HeapNode *hn;
-
-  GNUNET_assert (off == GCPP_get_length (path) - 1);
-  GNUNET_assert (cp == GCPP_get_peer_at_offset (path,
-                                                off));
-  if (NULL == cp->path_heap)
-  {
-    /* #GCP_drop_owned_paths() was already called, we cannot take new ones! */
-    GNUNET_assert (GNUNET_NO == force);
-    return NULL;
-  }
-  desirability = GCPP_get_desirability (path);
-  if (GNUNET_NO == force)
-  {
-    /* FIXME: desirability is not yet initialized; tricky! */
-    if (GNUNET_NO ==
-        GNUNET_CONTAINER_heap_peek2 (cp->path_heap,
-                                     (void **) &root,
-                                     &root_desirability))
-    {
-      root = NULL;
-      root_desirability = 0;
-    }
-
-    if ( (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) &&
-         (desirability < root_desirability) )
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Decided to not attach path %p to peer %s due to undesirability\n",
-           GCPP_2s (path),
-           GCP_2s (cp));
-      return NULL;
-    }
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Attaching path %s to peer %s (%s)\n",
-       GCPP_2s (path),
-       GCP_2s (cp),
-       (GNUNET_NO == force) ? "desirable" : "forced");
-
-  /* Yes, we'd like to add this path, add to our heap */
-  hn = GNUNET_CONTAINER_heap_insert (cp->path_heap,
-                                     path,
-                                     desirability);
-
-  /* Consider maybe dropping other paths because of the new one */
-  if ( (GNUNET_CONTAINER_heap_get_size (cp->path_heap) >=
-        2 * DESIRED_CONNECTIONS_PER_TUNNEL) &&
-       (NULL != cp->heap_cleanup_task) )
-    cp->heap_cleanup_task = GNUNET_SCHEDULER_add_now (&path_heap_cleanup,
-                                                      cp);
-  return hn;
-}
-
-
-/**
- * This peer can no longer own @a path as the path
- * has been extended and a peer further down the line
- * is now the new owner.
- *
- * @param cp old owner of the @a path
- * @param path path where the ownership is lost
- * @param hn note in @a cp's path heap that must be deleted
- */
-void
-GCP_detach_path (struct CadetPeer *cp,
-                 struct CadetPeerPath *path,
-                 struct GNUNET_CONTAINER_HeapNode *hn)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Detatching path %s from peer %s\n",
-       GCPP_2s (path),
-       GCP_2s (cp));
-  GNUNET_assert (path ==
-                 GNUNET_CONTAINER_heap_remove_node (hn));
-}
-
-
-/**
- * Add a @a connection to this @a cp.
- *
- * @param cp peer via which the @a connection goes
- * @param cc the connection to add
- */
-void
-GCP_add_connection (struct CadetPeer *cp,
-                    struct CadetConnection *cc)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Adding connection %s to peer %s\n",
-       GCC_2s (cc),
-       GCP_2s (cp));
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multishortmap_put (cp->connections,
-                                                     &GCC_get_id (cc)->connection_of_tunnel,
-                                                     cc,
-                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  if (NULL != cp->destroy_task)
-  {
-    GNUNET_SCHEDULER_cancel (cp->destroy_task);
-    cp->destroy_task = NULL;
-  }
-}
-
-
-/**
- * Remove a @a connection that went via this @a cp.
- *
- * @param cp peer via which the @a connection went
- * @param cc the connection to remove
- */
-void
-GCP_remove_connection (struct CadetPeer *cp,
-                       struct CadetConnection *cc)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Removing connection %s from peer %s\n",
-       GCC_2s (cc),
-       GCP_2s (cp));
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multishortmap_remove (cp->connections,
-                                                        &GCC_get_id (cc)->connection_of_tunnel,
-                                                        cc));
-  consider_peer_destroy (cp);
-}
-
-
-/**
- * Retrieve the CadetPeer stucture associated with the
- * peer. Optionally create one and insert it in the appropriate
- * structures if the peer is not known yet.
- *
- * @param peer_id Full identity of the peer.
- * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO to return NULL if peer is unknown.
- * @return Existing or newly created peer structure.
- *         NULL if unknown and not requested @a create
- */
-struct CadetPeer *
-GCP_get (const struct GNUNET_PeerIdentity *peer_id,
-         int create)
-{
-  struct CadetPeer *cp;
-
-  cp = GNUNET_CONTAINER_multipeermap_get (peers,
-                                          peer_id);
-  if (NULL != cp)
-    return cp;
-  if (GNUNET_NO == create)
-    return NULL;
-  cp = GNUNET_new (struct CadetPeer);
-  cp->pid = *peer_id;
-  cp->connections = GNUNET_CONTAINER_multishortmap_create (32,
-                                                           GNUNET_YES);
-  cp->path_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multipeermap_put (peers,
-                                                    &cp->pid,
-                                                    cp,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating peer %s\n",
-       GCP_2s (cp));
-  return cp;
-}
-
-
-/**
- * Obtain the peer identity for a `struct CadetPeer`.
- *
- * @param cp our peer handle
- * @return the peer identity
- */
-const struct GNUNET_PeerIdentity *
-GCP_get_id (struct CadetPeer *cp)
-{
-  return &cp->pid;
-}
-
-
-/**
- * Iterate over all known peers.
- *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
-                 void *cls)
-{
-  GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                         iter,
-                                         cls);
-}
-
-
-/**
- * Count the number of known paths toward the peer.
- *
- * @param cp Peer to get path info.
- * @return Number of known paths.
- */
-unsigned int
-GCP_count_paths (const struct CadetPeer *cp)
-{
-  return cp->num_paths;
-}
-
-
-/**
- * Iterate over the paths to a peer.
- *
- * @param cp Peer to get path info.
- * @param callback Function to call for every path.
- * @param callback_cls Closure for @a callback.
- * @return Number of iterated paths.
- */
-unsigned int
-GCP_iterate_paths (struct CadetPeer *cp,
-                   GCP_PathIterator callback,
-                   void *callback_cls)
-{
-  unsigned int ret = 0;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Iterating over paths to peer %s%s\n",
-       GCP_2s (cp),
-       (NULL == cp->core_mq) ? "" : " including direct link");
-  if (NULL != cp->core_mq)
-  {
-    struct CadetPeerPath *path;
-
-    path = GCPP_get_path_from_route (1,
-                                     &cp->pid);
-    ret++;
-    if (GNUNET_NO ==
-        callback (callback_cls,
-                  path,
-                  0))
-      return ret;
-  }
-  for (unsigned int i=0;i<cp->path_dll_length;i++)
-  {
-    for (struct CadetPeerPathEntry *pe = cp->path_heads[i];
-         NULL != pe;
-         pe = pe->next)
-    {
-      ret++;
-      if (GNUNET_NO ==
-          callback (callback_cls,
-                    pe->path,
-                    i))
-        return ret;
-    }
-  }
-  return ret;
-}
-
-
-/**
- * Iterate over the paths to @a cp where
- * @a cp is at distance @a dist from us.
- *
- * @param cp Peer to get path info.
- * @param dist desired distance of @a cp to us on the path
- * @param callback Function to call for every path.
- * @param callback_cls Closure for @a callback.
- * @return Number of iterated paths.
- */
-unsigned int
-GCP_iterate_paths_at (struct CadetPeer *cp,
-                      unsigned int dist,
-                      GCP_PathIterator callback,
-                      void *callback_cls)
-{
-  unsigned int ret = 0;
-
-  if (dist >= cp->path_dll_length)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Asked to look for paths at distance %u, but maximum for me is < %u\n",
-         dist,
-         cp->path_dll_length);
-    return 0;
-  }
-  for (struct CadetPeerPathEntry *pe = cp->path_heads[dist];
-       NULL != pe;
-       pe = pe->next)
-  {
-    if (GNUNET_NO ==
-        callback (callback_cls,
-                  pe->path,
-                  dist))
-      return ret;
-    ret++;
-  }
-  return ret;
-}
-
-
-/**
- * Get the tunnel towards a peer.
- *
- * @param cp Peer to get from.
- * @param create #GNUNET_YES to create a tunnel if we do not have one
- * @return Tunnel towards peer.
- */
-struct CadetTunnel *
-GCP_get_tunnel (struct CadetPeer *cp,
-                int create)
-{
-  if (NULL == cp)
-    return NULL;
-  if ( (NULL != cp->t) ||
-       (GNUNET_NO == create) )
-    return cp->t;
-  cp->t = GCT_create_tunnel (cp);
-  consider_peer_activate (cp);
-  return cp->t;
-}
-
-
-/**
- * Hello offer was passed to the transport service. Mark it
- * as done.
- *
- * @param cls the `struct CadetPeer` where the offer completed
- */
-static void
-hello_offer_done (void *cls)
-{
-  struct CadetPeer *cp = cls;
-
-  cp->hello_offer = NULL;
-}
-
-
-/**
- * We got a HELLO for a @a peer, remember it, and possibly
- * trigger adequate actions (like trying to connect).
- *
- * @param cp the peer we got a HELLO for
- * @param hello the HELLO to remember
- */
-void
-GCP_set_hello (struct CadetPeer *cp,
-               const struct GNUNET_HELLO_Message *hello)
-{
-  struct GNUNET_HELLO_Message *mrg;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got %u byte HELLO for peer %s\n",
-       (unsigned int) GNUNET_HELLO_size (hello),
-       GCP_2s (cp));
-  if (NULL != cp->hello_offer)
-  {
-    GNUNET_TRANSPORT_offer_hello_cancel (cp->hello_offer);
-    cp->hello_offer = NULL;
-  }
-  if (NULL != cp->hello)
-  {
-    mrg = GNUNET_HELLO_merge (hello,
-                              cp->hello);
-    GNUNET_free (cp->hello);
-    cp->hello = mrg;
-  }
-  else
-  {
-    cp->hello = GNUNET_memdup (hello,
-                               GNUNET_HELLO_size (hello));
-  }
-  cp->hello_offer
-    = GNUNET_TRANSPORT_offer_hello (cfg,
-                                    GNUNET_HELLO_get_header (cp->hello) ,
-                                    &hello_offer_done,
-                                    cp);
-  /* New HELLO means cp's destruction time may change... */
-  consider_peer_destroy (cp);
-}
-
-
-/**
- * The tunnel to the given peer no longer exists, remove it from our
- * data structures, and possibly clean up the peer itself.
- *
- * @param cp the peer affected
- * @param t the dead tunnel
- */
-void
-GCP_drop_tunnel (struct CadetPeer *cp,
-                 struct CadetTunnel *t)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Dropping tunnel %s to peer %s\n",
-       GCT_2s (t),
-       GCP_2s (cp));
-  GNUNET_assert (cp->t == t);
-  cp->t = NULL;
-  consider_peer_destroy (cp);
-}
-
-
-/**
- * Test if @a cp has a core-level connection
- *
- * @param cp peer to test
- * @return #GNUNET_YES if @a cp has a core-level connection
- */
-int
-GCP_has_core_connection (struct CadetPeer *cp)
-{
-  return (NULL != cp->core_mq) ? GNUNET_YES : GNUNET_NO;
-}
-
-
-/**
- * Start message queue change notifications.
- *
- * @param cp peer to notify for
- * @param cb function to call if mq becomes available or unavailable
- * @param cb_cls closure for @a cb
- * @return handle to cancel request
- */
-struct GCP_MessageQueueManager *
-GCP_request_mq (struct CadetPeer *cp,
-                GCP_MessageQueueNotificationCallback cb,
-                void *cb_cls)
-{
-  struct GCP_MessageQueueManager *mqm;
-
-  mqm = GNUNET_new (struct GCP_MessageQueueManager);
-  mqm->cb = cb;
-  mqm->cb_cls = cb_cls;
-  mqm->cp = cp;
-  GNUNET_CONTAINER_DLL_insert (cp->mqm_head,
-                               cp->mqm_tail,
-                               mqm);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating MQM %p for peer %s\n",
-       mqm,
-       GCP_2s (cp));
-  if (NULL != cp->core_mq)
-    cb (cb_cls,
-        GNUNET_YES);
-  return mqm;
-}
-
-
-/**
- * Stops message queue change notifications.
- *
- * @param mqm handle matching request to cancel
- * @param last_env final message to transmit, or NULL
- */
-void
-GCP_request_mq_cancel (struct GCP_MessageQueueManager *mqm,
-                       struct GNUNET_MQ_Envelope *last_env)
-{
-  struct CadetPeer *cp = mqm->cp;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying MQM %p for peer %s%s\n",
-       mqm,
-       GCP_2s (cp),
-       (NULL == last_env) ? "" : " with last ditch transmission");
-  if (NULL != mqm->env)
-    GNUNET_MQ_discard (mqm->env);
-  if (NULL != last_env)
-  {
-    if (NULL != cp->core_mq)
-    {
-      GNUNET_MQ_notify_sent (last_env,
-                             &mqm_send_done,
-                             cp);
-      GNUNET_MQ_send (cp->core_mq,
-                      last_env);
-    }
-    else
-    {
-      GNUNET_MQ_discard (last_env);
-    }
-  }
-  if (cp->mqm_ready_ptr == mqm)
-    cp->mqm_ready_ptr = mqm->next;
-  GNUNET_CONTAINER_DLL_remove (cp->mqm_head,
-                               cp->mqm_tail,
-                               mqm);
-  GNUNET_free (mqm);
-}
-
-
-/**
- * Send the message in @a env to @a cp, overriding queueing logic.
- * This function should only be used to send error messages outside
- * of flow and congestion control, similar to ICMP.  Note that
- * the envelope may be silently discarded as well.
- *
- * @param cp peer to send the message to
- * @param env envelope with the message to send
- */
-void
-GCP_send_ooo (struct CadetPeer *cp,
-              struct GNUNET_MQ_Envelope *env)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending message to %s out of management\n",
-       GCP_2s (cp));
-  if (NULL == cp->core_mq)
-  {
-    GNUNET_MQ_discard (env);
-    return;
-  }
-  GNUNET_MQ_notify_sent (env,
-                         &mqm_send_done,
-                         cp);
-  GNUNET_MQ_send (cp->core_mq,
-                  env);
-}
-
-
-
-
-/* end of gnunet-service-cadet-new_peer.c */
diff --git a/src/cadet/gnunet-service-cadet-new_peer.h b/src/cadet/gnunet-service-cadet-new_peer.h
deleted file mode 100644
index e1d6fc33a..000000000
--- a/src/cadet/gnunet-service-cadet-new_peer.h
+++ /dev/null
@@ -1,394 +0,0 @@
-
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2001-2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet-new_peer.h
- * @brief Information we track per peer.
- * @author Bartlomiej Polot
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_CADET_PEER_H
-#define GNUNET_SERVICE_CADET_PEER_H
-
-#include "gnunet-service-cadet-new.h"
-#include "gnunet_hello_lib.h"
-
-
-/**
- * Get the static string for a peer ID.
- *
- * @param peer Peer.
- *
- * @return Static string for it's ID.
- */
-const char *
-GCP_2s (const struct CadetPeer *peer);
-
-
-/**
- * Retrieve the CadetPeer stucture associated with the
- * peer. Optionally create one and insert it in the appropriate
- * structures if the peer is not known yet.
- *
- * @param peer_id Full identity of the peer.
- * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO to return NULL if peer is unknown.
- * @return Existing or newly created peer structure.
- *         NULL if unknown and not requested @a create
- */
-struct CadetPeer *
-GCP_get (const struct GNUNET_PeerIdentity *peer_id,
-         int create);
-
-
-/**
- * Calculate how desirable a path is for @a cp if
- * @a cp is at offset @a off in the path.
- *
- * @param cp a peer reachable via a path
- * @param off offset of @a cp in a path
- * @return score how useful a path is to reach @a cp,
- *         positive scores mean path is more desirable
- */
-double
-GCP_get_desirability_of_path (struct CadetPeer *cp,
-                              unsigned int off);
-
-
-/**
- * Obtain the peer identity for a `struct CadetPeer`.
- *
- * @param cp our peer handle
- * @return the peer identity
- */
-const struct GNUNET_PeerIdentity *
-GCP_get_id (struct CadetPeer *cp);
-
-
-/**
- * Iterate over all known peers.
- *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
-                 void *cls);
-
-
-/**
- * Count the number of known paths toward the peer.
- *
- * @param cp Peer to get path info.
- * @return Number of known paths.
- */
-unsigned int
-GCP_count_paths (const struct CadetPeer *cp);
-
-
-/**
- * Drop all paths owned by this peer, and do not
- * allow new ones to be added: We are shutting down.
- *
- * @param cp peer to drop paths to
- */
-void
-GCP_drop_owned_paths (struct CadetPeer *cp);
-
-
-/**
- * Peer path iterator.
- *
- * @param cls Closure.
- * @param path Path itself
- * @param off offset of the target peer in @a path
- * @return #GNUNET_YES if should keep iterating.
- *         #GNUNET_NO otherwise.
- */
-typedef int
-(*GCP_PathIterator) (void *cls,
-                     struct CadetPeerPath *path,
-                     unsigned int off);
-
-
-/**
- * Iterate over the paths to a peer.
- *
- * @param cp Peer to get path info.
- * @param callback Function to call for every path.
- * @param callback_cls Closure for @a callback.
- * @return Number of iterated paths.
- */
-unsigned int
-GCP_iterate_paths (struct CadetPeer *cp,
-                   GCP_PathIterator callback,
-                   void *callback_cls);
-
-
-/**
- * Iterate over the paths to @a peer where
- * @a peer is at distance @a dist from us.
- *
- * @param cp Peer to get path info.
- * @param dist desired distance of @a peer to us on the path
- * @param callback Function to call for every path.
- * @param callback_cls Closure for @a callback.
- * @return Number of iterated paths.
- */
-unsigned int
-GCP_iterate_paths_at (struct CadetPeer *cp,
-                      unsigned int dist,
-                      GCP_PathIterator callback,
-                      void *callback_cls);
-
-
-/**
- * Remove an entry from the DLL of all of the paths that this peer is on.
- *
- * @param cp peer to modify
- * @param entry an entry on a path
- * @param off offset of this peer on the path
- */
-void
-GCP_path_entry_remove (struct CadetPeer *cp,
-                       struct CadetPeerPathEntry *entry,
-                       unsigned int off);
-
-
-/**
- * Add an entry to the DLL of all of the paths that this peer is on.
- *
- * @param cp peer to modify
- * @param entry an entry on a path
- * @param off offset of this peer on the path
- */
-void
-GCP_path_entry_add (struct CadetPeer *cp,
-                    struct CadetPeerPathEntry *entry,
-                    unsigned int off);
-
-
-/**
- * Get the tunnel towards a peer.
- *
- * @param cp Peer to get from.
- * @param create #GNUNET_YES to create a tunnel if we do not have one
- * @return Tunnel towards peer.
- */
-struct CadetTunnel *
-GCP_get_tunnel (struct CadetPeer *cp,
-                int create);
-
-
-/**
- * The tunnel to the given peer no longer exists, remove it from our
- * data structures, and possibly clean up the peer itself.
- *
- * @param cp the peer affected
- * @param t the dead tunnel
- */
-void
-GCP_drop_tunnel (struct CadetPeer *cp,
-                 struct CadetTunnel *t);
-
-
-/**
- * Try adding a @a path to this @a cp.  If the peer already
- * has plenty of paths, return NULL.
- *
- * @param cp peer to which the @a path leads to
- * @param path a path looking for an owner; may not be fully initialized yet!
- * @param off offset of @a cp in @a path
- * @param force for attaching the path
- * @return NULL if this peer does not care to become a new owner,
- *         otherwise the node in the peer's path heap for the @a path.
- */
-struct GNUNET_CONTAINER_HeapNode *
-GCP_attach_path (struct CadetPeer *cp,
-                 struct CadetPeerPath *path,
-                 unsigned int off,
-                 int force);
-
-
-/**
- * This peer can no longer own @a path as the path
- * has been extended and a peer further down the line
- * is now the new owner.
- *
- * @param cp old owner of the @a path
- * @param path path where the ownership is lost
- * @param hn note in @a cp's path heap that must be deleted
- */
-void
-GCP_detach_path (struct CadetPeer *cp,
-                 struct CadetPeerPath *path,
-                 struct GNUNET_CONTAINER_HeapNode *hn);
-
-
-/**
- * Add a @a connection to this @a cp.
- *
- * @param cp peer via which the @a connection goes
- * @param cc the connection to add
- */
-void
-GCP_add_connection (struct CadetPeer *cp,
-                    struct CadetConnection *cc);
-
-
-/**
- * Remove a @a connection that went via this @a cp.
- *
- * @param cp peer via which the @a connection went
- * @param cc the connection to remove
- */
-void
-GCP_remove_connection (struct CadetPeer *cp,
-                       struct CadetConnection *cc);
-
-
-/**
- * We got a HELLO for a @a cp, remember it, and possibly
- * trigger adequate actions (like trying to connect).
- *
- * @param cp the peer we got a HELLO for
- * @param hello the HELLO to remember
- */
-void
-GCP_set_hello (struct CadetPeer *cp,
-               const struct GNUNET_HELLO_Message *hello);
-
-
-/**
- * Clean up all entries about all peers.
- * Must only be called after all tunnels, CORE-connections and
- * connections are down.
- */
-void
-GCP_destroy_all_peers (void);
-
-
-/**
- * Data structure used to track whom we have to notify about changes
- * in our ability to transmit to a given peer.
- *
- * All queue managers will be given equal chance for sending messages
- * to @a cp.  This construct this guarantees fairness for access to @a
- * cp among the different message queues.  Each connection or route
- * will have its respective message queue managers for each direction.
- */
-struct GCP_MessageQueueManager;
-
-
-/**
- * Function to call with updated message queue object.
- *
- * @param cls closure
- * @param available #GNUNET_YES if sending is now possible,
- *                  #GNUNET_NO if sending is no longer possible
- *                  #GNUNET_SYSERR if sending is no longer possible
- *                                 and the last envelope was discarded
- */
-typedef void
-(*GCP_MessageQueueNotificationCallback)(void *cls,
-                                        int available);
-
-
-/**
- * Start message queue change notifications.  Will create a new slot
- * to manage the message queue to the given @a cp.
- *
- * @param cp peer to notify for
- * @param cb function to call if mq becomes available or unavailable
- * @param cb_cls closure for @a cb
- * @return handle to cancel request
- */
-struct GCP_MessageQueueManager *
-GCP_request_mq (struct CadetPeer *cp,
-                GCP_MessageQueueNotificationCallback cb,
-                void *cb_cls);
-
-
-/**
- * Test if @a cp has a core-level connection
- *
- * @param cp peer to test
- * @return #GNUNET_YES if @a cp has a core-level connection
- */
-int
-GCP_has_core_connection (struct CadetPeer *cp);
-
-
-/**
- * Send the message in @a env via a @a mqm.  Must only be called at
- * most once after the respective
- * #GCP_MessageQueueNotificationCallback was called with `available`
- * set to #GNUNET_YES, and not after the callback was called with
- * `available` set to #GNUNET_NO or #GNUNET_SYSERR.
- *
- * @param mqm message queue manager for the transmission
- * @param env envelope with the message to send; must NOT
- *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
- */
-void
-GCP_send (struct GCP_MessageQueueManager *mqm,
-          struct GNUNET_MQ_Envelope *env);
-
-
-/**
- * Send the message in @a env to @a cp, overriding queueing logic.
- * This function should only be used to send error messages outside
- * of flow and congestion control, similar to ICMP.  Note that
- * the envelope may be silently discarded as well.
- *
- * @param cp peer to send the message to
- * @param env envelope with the message to send
- */
-void
-GCP_send_ooo (struct CadetPeer *cp,
-              struct GNUNET_MQ_Envelope *env);
-
-
-/**
- * Stops message queue change notifications and sends a last message.
- * In practice, this is implemented by sending that @a last_env
- * message immediately (if any), ignoring queue order.
- *
- * @param mqm handle matching request to cancel
- * @param last_env final message to transmit, or NULL
- */
-void
-GCP_request_mq_cancel (struct GCP_MessageQueueManager *mqm,
-                       struct GNUNET_MQ_Envelope *last_env);
-
-
-/**
- * Set the message queue to @a mq for peer @a cp and notify watchers.
- *
- * @param cp peer to modify
- * @param mq message queue to set (can be NULL)
- */
-void
-GCP_set_mq (struct CadetPeer *cp,
-            struct GNUNET_MQ_Handle *mq);
-
-
-#endif
diff --git a/src/cadet/gnunet-service-cadet.c b/src/cadet/gnunet-service-cadet.c
index 3a07f0ee5..a7e1fca47 100644
--- a/src/cadet/gnunet-service-cadet.c
+++ b/src/cadet/gnunet-service-cadet.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2001-2013 GNUnet e.V.
+     Copyright (C) 2001-2013, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -22,38 +22,82 @@
  * @file cadet/gnunet-service-cadet.c
  * @brief GNUnet CADET service with encryption
  * @author Bartlomiej Polot
- *
- *  FIXME in progress:
- * - rekey - reliability interaction
- * - channel retransmit timing
- *
- * TODO:
- * - relay corking down to core
- * - set ttl relative to path length
- * TODO END
+ * @author Christian Grothoff
  *
  * Dictionary:
  * - peer: other cadet instance. If there is direct connection it's a neighbor.
- * - tunnel: encrypted connection to a peer, neighbor or not.
- * - channel: connection between two clients, on the same or different peers.
- *            have properties like reliability.
  * - path: series of directly connected peer from one peer to another.
  * - connection: path which is being used in a tunnel.
+ * - tunnel: encrypted connection to a peer, neighbor or not.
+ * - channel: logical link between two clients, on the same or different peers.
+ *            have properties like reliability.
  */
 
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "cadet.h"
 #include "gnunet_statistics_service.h"
-
-#include "gnunet-service-cadet_local.h"
+#include "gnunet-service-cadet.h"
 #include "gnunet-service-cadet_channel.h"
 #include "gnunet-service-cadet_connection.h"
-#include "gnunet-service-cadet_tunnel.h"
+#include "gnunet-service-cadet_core.h"
 #include "gnunet-service-cadet_dht.h"
-#include "gnunet-service-cadet_peer.h"
 #include "gnunet-service-cadet_hello.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
+
+#define LOG(level, ...) GNUNET_log (level,__VA_ARGS__)
+
+
+/**
+ * Struct containing information about a client of the service
+ */
+struct CadetClient
+{
+  /**
+   * Linked list next
+   */
+  struct CadetClient *next;
+
+  /**
+   * Linked list prev
+   */
+  struct CadetClient *prev;
+
+  /**
+   * Tunnels that belong to this client, indexed by local id,
+   * value is a `struct CadetChannel`.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap32 *channels;
+
+  /**
+   * Handle to communicate with the client
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Client handle.
+   */
+  struct GNUNET_SERVICE_Client *client;
+
+  /**
+   * Ports that this client has declared interest in.
+   * Indexed by port, contains *Client.
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *ports;
 
+  /**
+   * Channel ID to use for the next incoming channel for this client.
+   * Wraps around (in theory).
+   */
+  struct GNUNET_CADET_ClientChannelNumber next_ccn;
+
+  /**
+   * ID of the client, mainly for debug messages. Purely internal to this file.
+   */
+  unsigned int id;
+};
 
 /******************************************************************************/
 /***********************      GLOBAL VARIABLES     ****************************/
@@ -62,37 +106,317 @@
 /****************************** Global variables ******************************/
 
 /**
+ * Handle to our configuration.
+ */
+const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+/**
  * Handle to the statistics service.
  */
 struct GNUNET_STATISTICS_Handle *stats;
 
 /**
- * Local peer own ID (memory efficient handle).
+ * Handle to communicate with ATS.
  */
-GNUNET_PEER_Id myid;
+struct GNUNET_ATS_ConnectivityHandle *ats_ch;
 
 /**
- * Local peer own ID (full value).
+ * Local peer own ID.
  */
 struct GNUNET_PeerIdentity my_full_id;
 
+/**
+ * Own private key.
+ */
+struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
 
 /**
- * Signal that shutdown is happening: prevent recover measures.
+ * Signal that shutdown is happening: prevent recovery measures.
  */
 int shutting_down;
 
-/*************************** Static global variables **************************/
+/**
+ * DLL with all the clients, head.
+ */
+static struct CadetClient *clients_head;
 
 /**
- * Own private key.
+ * DLL with all the clients, tail.
  */
-static struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
+static struct CadetClient *clients_tail;
 
+/**
+ * Next ID to assign to a client.
+ */
+static unsigned int next_client_id;
+
+/**
+ * All ports clients of this peer have opened.
+ */
+struct GNUNET_CONTAINER_MultiHashMap *open_ports;
+
+/**
+ * Map from ports to channels where the ports were closed at the
+ * time we got the inbound connection.
+ * Indexed by port, contains `struct CadetChannel`.
+ */
+struct GNUNET_CONTAINER_MultiHashMap *loose_channels;
+
+/**
+ * Map from PIDs to `struct CadetPeer` entries.
+ */
+struct GNUNET_CONTAINER_MultiPeerMap *peers;
+
+/**
+ * Map from `struct GNUNET_CADET_ConnectionTunnelIdentifier`
+ * hash codes to `struct CadetConnection` objects.
+ */
+struct GNUNET_CONTAINER_MultiShortmap *connections;
+
+/**
+ * How many messages are needed to trigger an AXOLOTL ratchet advance.
+ */
+unsigned long long ratchet_messages;
+
+/**
+ * How long until we trigger a ratched advance due to time.
+ */
+struct GNUNET_TIME_Relative ratchet_time;
+
+/**
+ * How frequently do we send KEEPALIVE messages on idle connections?
+ */
+struct GNUNET_TIME_Relative keepalive_period;
+
+/**
+ * Set to non-zero values to create random drops to test retransmissions.
+ */
+unsigned long long drop_percent;
+
+
+/**
+ * Send a message to a client.
+ *
+ * @param c client to get the message
+ * @param env envelope with the message
+ */
+void
+GSC_send_to_client (struct CadetClient *c,
+                    struct GNUNET_MQ_Envelope *env)
+{
+  GNUNET_MQ_send (c->mq,
+                  env);
+}
+
+
+/**
+ * Return identifier for a client as a string.
+ *
+ * @param c client to identify
+ * @return string for debugging
+ */
+const char *
+GSC_2s (struct CadetClient *c)
+{
+  static char buf[32];
+
+  GNUNET_snprintf (buf,
+                   sizeof (buf),
+                   "Client(%u)",
+                   c->id);
+  return buf;
+}
+
+
+/**
+ * Lookup channel of client @a c by @a ccn.
+ *
+ * @param c client to look in
+ * @param ccn channel ID to look up
+ * @return NULL if no such channel exists
+ */
+static struct CadetChannel *
+lookup_channel (struct CadetClient *c,
+                struct GNUNET_CADET_ClientChannelNumber ccn)
+{
+  return GNUNET_CONTAINER_multihashmap32_get (c->channels,
+                                              ntohl (ccn.channel_of_client));
+}
+
+
+/**
+ * Obtain the next LID to use for incoming connections to
+ * the given client.
+ *
+ * @param c client handle
+ */
+static struct GNUNET_CADET_ClientChannelNumber
+client_get_next_ccn (struct CadetClient *c)
+{
+  struct GNUNET_CADET_ClientChannelNumber ccn = c->next_ccn;
+
+  /* increment until we have a free one... */
+  while (NULL !=
+         lookup_channel (c,
+                         ccn))
+  {
+    ccn.channel_of_client
+      = htonl (1 + (ntohl (ccn.channel_of_client)));
+    if (ntohl (ccn.channel_of_client) >=
+        GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+      ccn.channel_of_client = htonl (0);
+  }
+  c->next_ccn.channel_of_client
+    = htonl (1 + (ntohl (ccn.channel_of_client)));
+  return ccn;
+}
+
+
+/**
+ * Bind incoming channel to this client, and notify client about
+ * incoming connection.  Caller is responsible for notifying the other
+ * peer about our acceptance of the channel.
+ *
+ * @param c client to bind to
+ * @param ch channel to be bound
+ * @param dest peer that establishes the connection
+ * @param port port number
+ * @param options options
+ * @return local channel number assigned to the new client
+ */
+struct GNUNET_CADET_ClientChannelNumber
+GSC_bind (struct CadetClient *c,
+          struct CadetChannel *ch,
+          struct CadetPeer *dest,
+          const struct GNUNET_HashCode *port,
+          uint32_t options)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalChannelCreateMessage *cm;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+
+  ccn = client_get_next_ccn (c);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_put (c->channels,
+                                                      ntohl (ccn.channel_of_client),
+                                                      ch,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Accepting incoming %s from %s on open port %s (%u), assigning ccn %X\n",
+       GCCH_2s (ch),
+       GCP_2s (dest),
+       GNUNET_h2s (port),
+       (uint32_t) ntohl (options),
+       (uint32_t) ntohl (ccn.channel_of_client));
+  /* notify local client about incoming connection! */
+  env = GNUNET_MQ_msg (cm,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
+  cm->ccn = ccn;
+  cm->port = *port;
+  cm->opt = htonl (options);
+  cm->peer = *GCP_get_id (dest);
+  GSC_send_to_client (c,
+                      env);
+  return ccn;
+}
+
+
+/**
+ * Callback invoked on all peers to destroy all tunnels
+ * that may still exist.
+ *
+ * @param cls NULL
+ * @param pid identify of a peer
+ * @param value a `struct CadetPeer` that may still have a tunnel
+ * @return #GNUNET_OK (iterate over all entries)
+ */
+static int
+destroy_tunnels_now (void *cls,
+                     const struct GNUNET_PeerIdentity *pid,
+                     void *value)
+{
+  struct CadetPeer *cp = value;
+  struct CadetTunnel *t = GCP_get_tunnel (cp,
+                                          GNUNET_NO);
+
+  if (NULL != t)
+    GCT_destroy_tunnel_now (t);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Callback invoked on all peers to destroy all tunnels
+ * that may still exist.
+ *
+ * @param cls NULL
+ * @param pid identify of a peer
+ * @param value a `struct CadetPeer` that may still have a tunnel
+ * @return #GNUNET_OK (iterate over all entries)
+ */
+static int
+destroy_paths_now (void *cls,
+                   const struct GNUNET_PeerIdentity *pid,
+                   void *value)
+{
+  struct CadetPeer *cp = value;
+
+  GCP_drop_owned_paths (cp);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Shutdown everything once the clients have disconnected.
+ */
+static void
+shutdown_rest ()
+{
+  if (NULL != stats)
+  {
+    GNUNET_STATISTICS_destroy (stats,
+                               GNUNET_NO);
+    stats = NULL;
+  }
+  if (NULL != open_ports)
+  {
+    GNUNET_CONTAINER_multihashmap_destroy (open_ports);
+    open_ports = NULL;
+  }
+  if (NULL != loose_channels)
+  {
+    GNUNET_CONTAINER_multihashmap_destroy (loose_channels);
+    loose_channels = NULL;
+  }
+  /* Destroy tunnels.  Note that all channels must be destroyed first! */
+  GCP_iterate_all (&destroy_tunnels_now,
+                   NULL);
+  /* All tunnels, channels, connections and CORE must be down before this point. */
+  GCP_iterate_all (&destroy_paths_now,
+                   NULL);
+  /* All paths, tunnels, channels, connections and CORE must be down before this point. */
+  GCP_destroy_all_peers ();
+  if (NULL != peers)
+  {
+    GNUNET_CONTAINER_multipeermap_destroy (peers);
+    peers = NULL;
+  }
+  if (NULL != connections)
+  {
+    GNUNET_CONTAINER_multishortmap_destroy (connections);
+    connections = NULL;
+  }
+  if (NULL != ats_ch)
+  {
+    GNUNET_ATS_connectivity_done (ats_ch);
+    ats_ch = NULL;
+  }
+  GCD_shutdown ();
+  GCH_shutdown ();
+  GNUNET_free_non_null (my_private_key);
+  my_private_key = NULL;
+}
 
-/******************************************************************************/
-/************************      MAIN FUNCTIONS      ****************************/
-/******************************************************************************/
 
 /**
  * Task run during shutdown.
@@ -102,83 +426,1071 @@ static struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
 static void
 shutdown_task (void *cls)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "shutting down\n");
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Shutting down\n");
   shutting_down = GNUNET_YES;
+  GCO_shutdown ();
+  if (NULL == clients_head)
+    shutdown_rest ();
+}
 
-  GML_shutdown ();
-  GCH_shutdown ();
-  GCC_shutdown ();
-  GCT_shutdown ();
-  GCD_shutdown ();
-  GCP_shutdown ();
 
-  GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
-  stats = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "shut down\n");
+/**
+ * We had a remote connection @a value to port @a port before
+ * client @a cls opened port @a port.  Bind them now.
+ *
+ * @param cls the `struct CadetClient`
+ * @param port the port
+ * @param value the `struct CadetChannel`
+ * @return #GNUNET_YES (iterate over all such channels)
+ */
+static int
+bind_loose_channel (void *cls,
+                    const struct GNUNET_HashCode *port,
+                    void *value)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch = value;
+
+  GCCH_bind (ch,
+             c);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (loose_channels,
+                                                       port,
+                                                       value));
+  return GNUNET_YES;
 }
 
 
 /**
- * Process cadet requests.
+ * Handle port open request.  Creates a mapping from the
+ * port to the respective client and checks whether we have
+ * loose channels trying to bind to the port.  If so, those
+ * are bound.
  *
- * @param cls closure
- * @param server the initialized server
- * @param c configuration to use
+ * @param cls Identification of the client.
+ * @param pmsg The actual message.
  */
 static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
-     const struct GNUNET_CONFIGURATION_Handle *c)
+handle_port_open (void *cls,
+                  const struct GNUNET_CADET_PortMessage *pmsg)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "starting to run\n");
+  struct CadetClient *c = cls;
 
-  stats = GNUNET_STATISTICS_create ("cadet", c);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Open port %s requested by %s\n",
+       GNUNET_h2s (&pmsg->port),
+       GSC_2s (c));
+  if (NULL == c->ports)
+    c->ports = GNUNET_CONTAINER_multihashmap_create (4,
+                                                      GNUNET_NO);
+  if (GNUNET_OK !=
+      GNUNET_CONTAINER_multihashmap_put (c->ports,
+                                         &pmsg->port,
+                                         c,
+                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  (void) GNUNET_CONTAINER_multihashmap_put (open_ports,
+                                            &pmsg->port,
+                                            c,
+                                            GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+  GNUNET_CONTAINER_multihashmap_get_multiple (loose_channels,
+                                              &pmsg->port,
+                                              &bind_loose_channel,
+                                              c);
+  GNUNET_SERVICE_client_continue (c->client);
+}
 
-  /* Scheduled the task to clean up when shutdown is called */
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
-                                 NULL);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "reading key\n");
-  my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (c);
-  GNUNET_assert (NULL != my_private_key);
-  GNUNET_CRYPTO_eddsa_key_get_public (my_private_key, &my_full_id.public_key);
-  myid = GNUNET_PEER_intern (&my_full_id);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "STARTING SERVICE (cadet) for peer [%s]\n",
-              GNUNET_i2s (&my_full_id));
 
-  GML_init (server);    /* Local clients */
-  GCH_init (c);         /* Hellos */
-  GCC_init (c);         /* Connections */
-  GCP_init (c);         /* Peers */
-  GCD_init (c);         /* DHT */
-  GCT_init (c, my_private_key); /* Tunnels */
+/**
+ * Handler for port close requests.  Marks this port as closed
+ * (unless of course we have another client with the same port
+ * open).  Note that existing channels accepted on the port are
+ * not affected.
+ *
+ * @param cls Identification of the client.
+ * @param pmsg The actual message.
+ */
+static void
+handle_port_close (void *cls,
+                   const struct GNUNET_CADET_PortMessage *pmsg)
+{
+  struct CadetClient *c = cls;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Cadet service running\n");
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Closing port %s as requested by %s\n",
+       GNUNET_h2s (&pmsg->port),
+       GSC_2s (c));
+  if (GNUNET_YES !=
+      GNUNET_CONTAINER_multihashmap_remove (c->ports,
+                                            &pmsg->port,
+                                            c))
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (open_ports,
+                                                       &pmsg->port,
+                                                       c));
+  GNUNET_SERVICE_client_continue (c->client);
 }
 
 
 /**
- * The main function for the cadet service.
+ * Handler for requests for us creating a new channel to another peer and port.
  *
- * @param argc number of arguments from the command line
- * @param argv command line arguments
- * @return 0 ok, 1 on error
+ * @param cls Identification of the client.
+ * @param tcm The actual message.
  */
-int
-main (int argc, char *const *argv)
+static void
+handle_channel_create (void *cls,
+                       const struct GNUNET_CADET_LocalChannelCreateMessage *tcm)
 {
-  int r;
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
 
-  shutting_down = GNUNET_NO;
-  r = GNUNET_SERVICE_run (argc, argv, "cadet", GNUNET_SERVICE_OPTION_NONE, &run,
-                          NULL);
-  GNUNET_free (my_private_key);
+  if (ntohl (tcm->ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+  {
+    /* Channel ID not in allowed range. */
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  ch = lookup_channel (c,
+                       tcm->ccn);
+  if (NULL != ch)
+  {
+    /* Channel ID already in use. Not allowed. */
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "New channel to %s at port %s requested by %s\n",
+       GNUNET_i2s (&tcm->peer),
+       GNUNET_h2s (&tcm->port),
+       GSC_2s (c));
 
-  if (GNUNET_OK != r)
+  /* Create channel */
+  ch = GCCH_channel_local_new (c,
+                               tcm->ccn,
+                               GCP_get (&tcm->peer,
+                                        GNUNET_YES),
+                               &tcm->port,
+                               ntohl (tcm->opt));
+  if (NULL == ch)
   {
-    FPRINTF (stderr, "GNUNET_SERVICE_run for CADET has failed!\n");
-    return 1;
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
   }
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_put (c->channels,
+                                                      ntohl (tcm->ccn.channel_of_client),
+                                                      ch,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 
-  return 0;
+  GNUNET_SERVICE_client_continue (c->client);
 }
+
+
+/**
+ * Handler for requests of destroying an existing channel.
+ *
+ * @param cls client identification of the client
+ * @param msg the actual message
+ */
+static void
+handle_channel_destroy (void *cls,
+                        const struct GNUNET_CADET_LocalChannelDestroyMessage *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (c,
+                       msg->ccn);
+  if (NULL == ch)
+  {
+    /* Client attempted to destroy unknown channel.
+       Can happen if the other side went down at the same time.*/
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "%s tried to destroy unknown channel %X\n",
+         GSC_2s(c),
+         (uint32_t) ntohl (msg->ccn.channel_of_client));
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s is destroying %s\n",
+       GSC_2s(c),
+       GCCH_2s (ch));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
+                                                         ntohl (msg->ccn.channel_of_client),
+                                                         ch));
+  GCCH_channel_local_destroy (ch,
+                              c,
+                              msg->ccn);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Check for client traffic data message is well-formed.
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ * @return #GNUNET_OK if @a msg is OK, #GNUNET_SYSERR if not
+ */
+static int
+check_local_data (void *cls,
+                  const struct GNUNET_CADET_LocalData *msg)
+{
+  size_t payload_size;
+  size_t payload_claimed_size;
+  const char *buf;
+  struct GNUNET_MessageHeader pa;
+
+  /* FIXME: what is the format we shall allow for @a msg?
+     ONE payload item or multiple? Seems current cadet_api
+     at least in theory allows more than one. Next-gen
+     cadet_api will likely no more, so we could then
+     simplify this mess again. */
+  /* Sanity check for message size */
+  payload_size = ntohs (msg->header.size) - sizeof (*msg);
+  buf = (const char *) &msg[1];
+  while (payload_size >= sizeof (struct GNUNET_MessageHeader))
+  {
+    /* need to memcpy() for alignment */
+    GNUNET_memcpy (&pa,
+                   buf,
+                   sizeof (pa));
+    payload_claimed_size = ntohs (pa.size);
+    if ( (payload_size < payload_claimed_size) ||
+         (payload_claimed_size < sizeof (struct GNUNET_MessageHeader)) ||
+         (GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE < payload_claimed_size) )
+    {
+      GNUNET_break (0);
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Local data of %u total size had sub-message %u at %u with %u bytes\n",
+           ntohs (msg->header.size),
+           ntohs (pa.type),
+           (unsigned int) (buf - (const char *) &msg[1]),
+           (unsigned int) payload_claimed_size);
+      return GNUNET_SYSERR;
+    }
+    payload_size -= payload_claimed_size;
+    buf += payload_claimed_size;
+  }
+  if (0 != payload_size)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handler for client payload traffic to be send on a channel to
+ * another peer.
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ */
+static void
+handle_local_data (void *cls,
+                   const struct GNUNET_CADET_LocalData *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+  size_t payload_size;
+  const char *buf;
+
+  ch = lookup_channel (c,
+                       msg->ccn);
+  if (NULL == ch)
+  {
+    /* Channel does not exist (anymore) */
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Dropping payload for channel %u from client (channel unknown, other endpoint may have disconnected)\n",
+         (unsigned int) ntohl (msg->ccn.channel_of_client));
+    GNUNET_SERVICE_client_continue (c->client);
+    return;
+  }
+  payload_size = ntohs (msg->header.size) - sizeof (*msg);
+  GNUNET_STATISTICS_update (stats,
+                            "# payload received from clients",
+                            payload_size,
+                            GNUNET_NO);
+  buf = (const char *) &msg[1];
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received %u bytes payload from %s for %s\n",
+       (unsigned int) payload_size,
+       GSC_2s (c),
+       GCCH_2s (ch));
+  if (GNUNET_OK !=
+      GCCH_handle_local_data (ch,
+                              msg->ccn,
+                              buf,
+                              payload_size))
+  {
+    GNUNET_SERVICE_client_drop (c->client);
+    return;
+  }
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Handler for client's ACKs for payload traffic.
+ *
+ * @param cls identification of the client.
+ * @param msg The actual message.
+ */
+static void
+handle_local_ack (void *cls,
+                  const struct GNUNET_CADET_LocalAck *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetChannel *ch;
+
+  ch = lookup_channel (c,
+                       msg->ccn);
+  if (NULL == ch)
+  {
+    /* Channel does not exist (anymore) */
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Ignoring local ACK for channel %u from client (channel unknown, other endpoint may have disconnected)\n",
+         (unsigned int) ntohl (msg->ccn.channel_of_client));
+    GNUNET_SERVICE_client_continue (c->client);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got a local ACK from %s for %s\n",
+       GSC_2s(c),
+       GCCH_2s (ch));
+  GCCH_handle_local_ack (ch,
+                         msg->ccn);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all peers to send a monitoring client info about each peer.
+ *
+ * @param cls Closure ().
+ * @param peer Peer ID (tunnel remote peer).
+ * @param value Peer info.
+ * @return #GNUNET_YES, to keep iterating.
+ */
+static int
+get_all_peers_iterator (void *cls,
+                        const struct GNUNET_PeerIdentity *peer,
+                        void *value)
+{
+  struct CadetClient *c = cls;
+  struct CadetPeer *p = value;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalInfoPeer *msg;
+
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
+  msg->destination = *peer;
+  msg->paths = htons (GCP_count_paths (p));
+  msg->tunnel = htons (NULL != GCP_get_tunnel (p,
+                                               GNUNET_NO));
+  GNUNET_MQ_send (c->mq,
+                  env);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's INFO PEERS request.
+ *
+ * @param cls Identification of the client.
+ * @param message The actual message.
+ */
+static void
+handle_get_peers (void *cls,
+                  const struct GNUNET_MessageHeader *message)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *reply;
+
+  GCP_iterate_all (&get_all_peers_iterator,
+                   c);
+  env = GNUNET_MQ_msg (reply,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all paths of a peer to build an InfoPeer message.
+ * Message contains blocks of peers, first not included.
+ *
+ * @param cls message queue for transmission
+ * @param path Path itself
+ * @param off offset of the peer on @a path
+ * @return #GNUNET_YES if should keep iterating.
+ *         #GNUNET_NO otherwise.
+ */
+static int
+path_info_iterator (void *cls,
+                    struct CadetPeerPath *path,
+                    unsigned int off)
+{
+  struct GNUNET_MQ_Handle *mq = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *resp;
+  struct GNUNET_PeerIdentity *id;
+  uint16_t path_size;
+  unsigned int i;
+  unsigned int path_length;
+
+  path_length = GCPP_get_length (path);
+  path_size = sizeof (struct GNUNET_PeerIdentity) * (path_length - 1);
+  if (sizeof (*resp) + path_size > UINT16_MAX)
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Path of %u entries is too long for info message\n",
+         path_length);
+    return GNUNET_YES;
+  }
+  env = GNUNET_MQ_msg_extra (resp,
+                             path_size,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
+  id = (struct GNUNET_PeerIdentity *) &resp[1];
+
+  /* Don't copy first peer.  First peer is always the local one.  Last
+   * peer is always the destination (leave as 0, EOL).
+   */
+  for (i = 0; i < off; i++)
+    id[i] = *GCP_get_id (GCPP_get_peer_at_offset (path,
+                                                  i + 1));
+  GNUNET_MQ_send (mq,
+                  env);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's SHOW_PEER request.
+ *
+ * @param cls Identification of the client.
+ * @param msg The actual message.
+ */
+static void
+handle_show_peer (void *cls,
+                  const struct GNUNET_CADET_LocalInfo *msg)
+{
+  struct CadetClient *c = cls;
+  struct CadetPeer *p;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *resp;
+
+  p = GCP_get (&msg->peer,
+               GNUNET_NO);
+  if (NULL != p)
+    GCP_iterate_paths (p,
+                       &path_info_iterator,
+                       c->mq);
+  /* Send message with 0/0 to indicate the end */
+  env = GNUNET_MQ_msg (resp,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER_END);
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all tunnels to send a monitoring client info about each tunnel.
+ *
+ * @param cls Closure ().
+ * @param peer Peer ID (tunnel remote peer).
+ * @param value a `struct CadetPeer`
+ * @return #GNUNET_YES, to keep iterating.
+ */
+static int
+get_all_tunnels_iterator (void *cls,
+                          const struct GNUNET_PeerIdentity *peer,
+                          void *value)
+{
+  struct CadetClient *c = cls;
+  struct CadetPeer *p = value;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalInfoTunnel *msg;
+  struct CadetTunnel *t;
+
+  t = GCP_get_tunnel (p,
+                      GNUNET_NO);
+  if (NULL == t)
+    return GNUNET_YES;
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
+  msg->destination = *peer;
+  msg->channels = htonl (GCT_count_channels (t));
+  msg->connections = htonl (GCT_count_any_connections (t));
+  msg->cstate = htons (0);
+  msg->estate = htons ((uint16_t) GCT_get_estate (t));
+  GNUNET_MQ_send (c->mq,
+                  env);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS request.
+ *
+ * @param cls client Identification of the client.
+ * @param message The actual message.
+ */
+static void
+handle_info_tunnels (void *cls,
+                     const struct GNUNET_MessageHeader *message)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *reply;
+
+  GCP_iterate_all (&get_all_tunnels_iterator,
+                   c);
+  env = GNUNET_MQ_msg (reply,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Update the message with information about the connection.
+ *
+ * @param cls a `struct GNUNET_CADET_LocalInfoTunnel` message to update
+ * @param ct a connection about which we should store information in @a cls
+ */
+static void
+iter_connection (void *cls,
+                 struct CadetTConnection *ct)
+{
+  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
+  struct CadetConnection *cc = ct->cc;
+  struct GNUNET_CADET_ConnectionTunnelIdentifier *h;
+
+  h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
+  h[msg->connections++] = *(GCC_get_id (cc));
+}
+
+
+/**
+ * Update the message with information about the channel.
+ *
+ * @param cls a `struct GNUNET_CADET_LocalInfoTunnel` message to update
+ * @param ch a channel about which we should store information in @a cls
+ */
+static void
+iter_channel (void *cls,
+              struct CadetChannel *ch)
+{
+  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
+  struct GNUNET_CADET_ConnectionTunnelIdentifier *h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
+  struct GNUNET_CADET_ChannelTunnelNumber *chn
+    = (struct GNUNET_CADET_ChannelTunnelNumber *) &h[msg->connections];
+
+  chn[msg->channels++] = GCCH_get_id (ch);
+}
+
+
+/**
+ * Handler for client's #GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL request.
+ *
+ * @param cls Identification of the client.
+ * @param msg The actual message.
+ */
+static void
+handle_info_tunnel (void *cls,
+                    const struct GNUNET_CADET_LocalInfo *msg)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalInfoTunnel *resp;
+  struct CadetTunnel *t;
+  struct CadetPeer *p;
+  unsigned int ch_n;
+  unsigned int c_n;
+
+  p = GCP_get (&msg->peer,
+               GNUNET_NO);
+  t = GCP_get_tunnel (p,
+                      GNUNET_NO);
+  if (NULL == t)
+  {
+    /* We don't know the tunnel */
+    struct GNUNET_MQ_Envelope *env;
+    struct GNUNET_CADET_LocalInfoTunnel *warn;
+
+    LOG (GNUNET_ERROR_TYPE_INFO,
+         "Tunnel to %s unknown\n",
+         GNUNET_i2s_full (&msg->peer));
+    env = GNUNET_MQ_msg (warn,
+                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
+    warn->destination = msg->peer;
+    GNUNET_MQ_send (c->mq,
+                    env);
+    GNUNET_SERVICE_client_continue (c->client);
+    return;
+  }
+
+  /* Initialize context */
+  ch_n = GCT_count_channels (t);
+  c_n = GCT_count_any_connections (t);
+  env = GNUNET_MQ_msg_extra (resp,
+                             c_n * sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier) +
+                             ch_n * sizeof (struct GNUNET_CADET_ChannelTunnelNumber),
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
+  resp->destination = msg->peer;
+  /* Do not reorder! #iter_channel needs counters in HBO! */
+  GCT_iterate_connections (t,
+                           &iter_connection,
+                           resp);
+  GCT_iterate_channels (t,
+                        &iter_channel,
+                        resp);
+  resp->connections = htonl (resp->connections);
+  resp->channels = htonl (resp->channels);
+  resp->cstate = htons (0);
+  resp->estate = htons (GCT_get_estate (t));
+  GNUNET_MQ_send (c->mq,
+                  env);
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+/**
+ * Iterator over all peers to dump info for each peer.
+ *
+ * @param cls Closure (unused).
+ * @param peer Peer ID (tunnel remote peer).
+ * @param value Peer info.
+ *
+ * @return #GNUNET_YES, to keep iterating.
+ */
+static int
+show_peer_iterator (void *cls,
+                    const struct GNUNET_PeerIdentity *peer,
+                    void *value)
+{
+  struct CadetPeer *p = value;
+  struct CadetTunnel *t;
+
+  t = GCP_get_tunnel (p,
+                      GNUNET_NO);
+  if (NULL != t)
+    GCT_debug (t,
+               GNUNET_ERROR_TYPE_ERROR);
+  LOG (GNUNET_ERROR_TYPE_ERROR, "\n");
+  return GNUNET_YES;
+}
+
+
+/**
+ * Handler for client's INFO_DUMP request.
+ *
+ * @param cls Identification of the client.
+ * @param message The actual message.
+ */
+static void
+handle_info_dump (void *cls,
+                  const struct GNUNET_MessageHeader *message)
+{
+  struct CadetClient *c = cls;
+
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Received dump info request from client %u\n",
+       c->id);
+
+  LOG (GNUNET_ERROR_TYPE_ERROR,
+       "*************************** DUMP START ***************************\n");
+  for (struct CadetClient *ci = clients_head;
+       NULL != ci;
+       ci = ci->next)
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         "Client %u (%p), handle: %p, ports: %u, channels: %u\n",
+         ci->id,
+         ci,
+         ci->client,
+         (NULL != c->ports)
+         ? GNUNET_CONTAINER_multihashmap_size (ci->ports)
+         : 0,
+         GNUNET_CONTAINER_multihashmap32_size (ci->channels));
+  }
+  LOG (GNUNET_ERROR_TYPE_ERROR, "***************************\n");
+  GCP_iterate_all (&show_peer_iterator,
+                   NULL);
+
+  LOG (GNUNET_ERROR_TYPE_ERROR,
+       "**************************** DUMP END ****************************\n");
+
+  GNUNET_SERVICE_client_continue (c->client);
+}
+
+
+
+/**
+ * Callback called when a client connects to the service.
+ *
+ * @param cls closure for the service
+ * @param client the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
+ */
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *client,
+                   struct GNUNET_MQ_Handle *mq)
+{
+  struct CadetClient *c;
+
+  c = GNUNET_new (struct CadetClient);
+  c->client = client;
+  c->mq = mq;
+  c->id = next_client_id++; /* overflow not important: just for debug */
+  c->channels
+    = GNUNET_CONTAINER_multihashmap32_create (32);
+  GNUNET_CONTAINER_DLL_insert (clients_head,
+                               clients_tail,
+                               c);
+  GNUNET_STATISTICS_update (stats,
+                            "# clients",
+                            +1,
+                            GNUNET_NO);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s connected\n",
+       GSC_2s (c));
+  return c;
+}
+
+
+/**
+ * A channel was destroyed by the other peer. Tell our client.
+ *
+ * @param c client that lost a channel
+ * @param ccn channel identification number for the client
+ * @param ch the channel object
+ */
+void
+GSC_handle_remote_channel_destroy (struct CadetClient *c,
+                                   struct GNUNET_CADET_ClientChannelNumber ccn,
+                                   struct CadetChannel *ch)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalChannelDestroyMessage *tdm;
+
+  env = GNUNET_MQ_msg (tdm,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
+  tdm->ccn = ccn;
+  GSC_send_to_client (c,
+                      env);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
+                                                         ntohl (ccn.channel_of_client),
+                                                         ch));
+}
+
+
+/**
+ * A client that created a loose channel that was not bound to a port
+ * disconnected, drop it from the #loose_channels list.
+ *
+ * @param port the port the channel was trying to bind to
+ * @param ch the channel that was lost
+ */
+void
+GSC_drop_loose_channel (const struct GNUNET_HashCode *port,
+                        struct CadetChannel *ch)
+{
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (loose_channels,
+                                                       port,
+                                                       ch));
+}
+
+
+/**
+ * Iterator for deleting each channel whose client endpoint disconnected.
+ *
+ * @param cls Closure (client that has disconnected).
+ * @param key The local channel id in host byte order
+ * @param value The value stored at the key (channel to destroy).
+ * @return #GNUNET_OK, keep iterating.
+ */
+static int
+channel_destroy_iterator (void *cls,
+                          uint32_t key,
+                          void *value)
+{
+  struct CadetClient *c = cls;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+  struct CadetChannel *ch = value;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying %s, due to %s disconnecting.\n",
+       GCCH_2s (ch),
+       GSC_2s (c));
+  ccn.channel_of_client = htonl (key);
+  GCCH_channel_local_destroy (ch,
+                              c,
+                              ccn);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (c->channels,
+                                                         key,
+                                                         ch));
+  return GNUNET_OK;
+}
+
+
+/**
+ * Remove client's ports from the global hashmap on disconnect.
+ *
+ * @param cls Closure (unused).
+ * @param key the port.
+ * @param value the `struct CadetClient` to remove
+ * @return #GNUNET_OK, keep iterating.
+ */
+static int
+client_release_ports (void *cls,
+                      const struct GNUNET_HashCode *key,
+                      void *value)
+{
+  struct CadetClient *c = value;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Closing port %s due to %s disconnect.\n",
+       GNUNET_h2s (key),
+       GSC_2s (c));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (open_ports,
+                                                       key,
+                                                       value));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (c->ports,
+                                                       key,
+                                                       value));
+  return GNUNET_OK;
+}
+
+
+/**
+ * Callback called when a client disconnected from the service
+ *
+ * @param cls closure for the service
+ * @param client the client that disconnected
+ * @param internal_cls should be equal to @a c
+ */
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *client,
+                      void *internal_cls)
+{
+  struct CadetClient *c = internal_cls;
+
+  GNUNET_assert (c->client == client);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "%s is disconnecting.\n",
+       GSC_2s (c));
+  if (NULL != c->channels)
+  {
+    GNUNET_CONTAINER_multihashmap32_iterate (c->channels,
+                                             &channel_destroy_iterator,
+                                             c);
+    GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap32_size (c->channels));
+    GNUNET_CONTAINER_multihashmap32_destroy (c->channels);
+  }
+  if (NULL != c->ports)
+  {
+    GNUNET_CONTAINER_multihashmap_iterate (c->ports,
+                                           &client_release_ports,
+                                           c);
+    GNUNET_CONTAINER_multihashmap_destroy (c->ports);
+  }
+  GNUNET_CONTAINER_DLL_remove (clients_head,
+                               clients_tail,
+                               c);
+  GNUNET_STATISTICS_update (stats,
+                            "# clients",
+                            -1,
+                            GNUNET_NO);
+  GNUNET_free (c);
+  if ( (NULL == clients_head) &&
+       (GNUNET_YES == shutting_down) )
+    shutdown_rest ();
+}
+
+
+/**
+ * Setup CADET internals.
+ *
+ * @param cls closure
+ * @param server the initialized server
+ * @param c configuration to use
+ */
+static void
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_SERVICE_Handle *service)
+{
+  cfg = c;
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
+                                             "RATCHET_MESSAGES",
+                                             &ratchet_messages))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "RATCHET_MESSAGES",
+                               "needs to be a number");
+    ratchet_messages = 64;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (c,
+                                           "CADET",
+                                           "RATCHET_TIME",
+                                           &ratchet_time))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "RATCHET_TIME",
+                               "need delay value");
+    ratchet_time = GNUNET_TIME_UNIT_HOURS;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (c,
+                                           "CADET",
+                                           "REFRESH_CONNECTION_TIME",
+                                           &keepalive_period))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "REFRESH_CONNECTION_TIME",
+                               "need delay value");
+    keepalive_period = GNUNET_TIME_UNIT_MINUTES;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
+                                             "DROP_PERCENT",
+                                             &drop_percent))
+  {
+    drop_percent = 0;
+  }
+  else
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "Cadet is running with DROP enabled.\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "This is NOT a good idea!\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "Remove DROP_PERCENT from config file.\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
+  }
+  my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (c);
+  if (NULL == my_private_key)
+  {
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  GNUNET_CRYPTO_eddsa_key_get_public (my_private_key,
+                                      &my_full_id.public_key);
+  stats = GNUNET_STATISTICS_create ("cadet",
+                                    c);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
+  ats_ch = GNUNET_ATS_connectivity_init (c);
+  /* FIXME: optimize code to allow GNUNET_YES here! */
+  open_ports = GNUNET_CONTAINER_multihashmap_create (16,
+                                                     GNUNET_NO);
+  loose_channels = GNUNET_CONTAINER_multihashmap_create (16,
+                                                         GNUNET_NO);
+  peers = GNUNET_CONTAINER_multipeermap_create (16,
+                                                GNUNET_YES);
+  connections = GNUNET_CONTAINER_multishortmap_create (256,
+                                                       GNUNET_YES);
+  GCH_init (c);
+  GCD_init (c);
+  GCO_init (c);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "CADET started for peer %s\n",
+              GNUNET_i2s (&my_full_id));
+
+}
+
+
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+("cadet",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_fixed_size (port_open,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN,
+                          struct GNUNET_CADET_PortMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (port_close,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE,
+                          struct GNUNET_CADET_PortMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (channel_create,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE,
+                          struct GNUNET_CADET_LocalChannelCreateMessage,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (channel_destroy,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY,
+                          struct GNUNET_CADET_LocalChannelDestroyMessage,
+                          NULL),
+ GNUNET_MQ_hd_var_size (local_data,
+                        GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
+                        struct GNUNET_CADET_LocalData,
+                        NULL),
+ GNUNET_MQ_hd_fixed_size (local_ack,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
+                          struct GNUNET_CADET_LocalAck,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (get_peers,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
+                          struct GNUNET_MessageHeader,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (show_peer,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
+                          struct GNUNET_CADET_LocalInfo,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (info_tunnels,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
+                          struct GNUNET_MessageHeader,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (info_tunnel,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
+                          struct GNUNET_CADET_LocalInfo,
+                          NULL),
+ GNUNET_MQ_hd_fixed_size (info_dump,
+                          GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP,
+                          struct GNUNET_MessageHeader,
+                          NULL),
+ GNUNET_MQ_handler_end ());
+
+/* end of gnunet-service-cadet-new.c */
diff --git a/src/cadet/gnunet-service-cadet-new.h b/src/cadet/gnunet-service-cadet.h
index bee5c67cc..2f2d7baf3 100644
--- a/src/cadet/gnunet-service-cadet-new.h
+++ b/src/cadet/gnunet-service-cadet.h
@@ -20,7 +20,7 @@
 */
 
 /**
- * @file cadet/gnunet-service-cadet-new.h
+ * @file cadet/gnunet-service-cadet.h
  * @brief Information we track per peer.
  * @author Bartlomiej Polot
  * @author Christian Grothoff
diff --git a/src/cadet/gnunet-service-cadet_channel.c b/src/cadet/gnunet-service-cadet_channel.c
index 7b7c6e57c..68e29b66b 100644
--- a/src/cadet/gnunet-service-cadet_channel.c
+++ b/src/cadet/gnunet-service-cadet_channel.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,31 +17,63 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
-
-
+/**
+ * @file cadet/gnunet-service-cadet_channel.c
+ * @brief logical links between CADET clients
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - Congestion/flow control:
+ *   + estimate max bandwidth using bursts and use to for CONGESTION CONTROL!
+ *     (and figure out how/where to use this!)
+ *   + figure out flow control without ACKs (unreliable traffic!)
+ * - revisit handling of 'unbuffered' traffic!
+ *   (need to push down through tunnel into connection selection)
+ * - revisit handling of 'buffered' traffic: 4 is a rather small buffer; maybe
+ *   reserve more bits in 'options' to allow for buffer size control?
+ */
 #include "platform.h"
-#include "gnunet_util_lib.h"
-
+#include "cadet.h"
 #include "gnunet_statistics_service.h"
+#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_paths.h"
 
-#include "cadet.h"
-#include "cadet_protocol.h"
+#define LOG(level,...) GNUNET_log_from (level,"cadet-chn",__VA_ARGS__)
 
-#include "gnunet-service-cadet_channel.h"
-#include "gnunet-service-cadet_local.h"
-#include "gnunet-service-cadet_tunnel.h"
-#include "gnunet-service-cadet_peer.h"
+/**
+ * How long do we initially wait before retransmitting?
+ */
+#define CADET_INITIAL_RETRANSMIT_TIME GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 250)
 
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-chn",__VA_ARGS__)
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-chn",__VA_ARGS__)
+/**
+ * How long do we wait before dropping state about incoming
+ * connection to closed port?
+ */
+#define TIMEOUT_CLOSED_PORT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 30)
 
-#define CADET_RETRANSMIT_TIME    GNUNET_TIME_relative_multiply(\
-                                    GNUNET_TIME_UNIT_MILLISECONDS, 250)
-#define CADET_RETRANSMIT_MARGIN  4
+/**
+ * How long do we wait at least before retransmitting ever?
+ */
+#define MIN_RTT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 75)
+
+/**
+ * Maximum message ID into the future we accept for out-of-order messages.
+ * If the message is more than this into the future, we drop it.  This is
+ * important both to detect values that are actually in the past, as well
+ * as to limit adversarially triggerable memory consumption.
+ *
+ * Note that right now we have "max_pending_messages = 4" hard-coded in
+ * the logic below, so a value of 4 would suffice here. But we plan to
+ * allow larger windows in the future...
+ */
+#define MAX_OUT_OF_ORDER_DISTANCE 1024
 
 
 /**
- * All the states a connection can be in.
+ * All the states a channel can be in.
  */
 enum CadetChannelState
 {
@@ -51,9 +83,15 @@ enum CadetChannelState
   CADET_CHANNEL_NEW,
 
   /**
-   * Connection create message sent, waiting for ACK.
+   * Channel is to a port that is not open, we're waiting for the
+   * port to be opened.
+   */
+  CADET_CHANNEL_LOOSE,
+
+  /**
+   * CHANNEL_OPEN message sent, waiting for CHANNEL_OPEN_ACK.
    */
-  CADET_CHANNEL_SENT,
+  CADET_CHANNEL_OPEN_SENT,
 
   /**
    * Connection confirmed, ready to carry traffic.
@@ -63,138 +101,144 @@ enum CadetChannelState
 
 
 /**
- * Info holder for channel messages in queues.
+ * Info needed to retry a message in case it gets lost.
+ * Note that we DO use this structure also for unreliable
+ * messages.
  */
-struct CadetChannelQueue
+struct CadetReliableMessage
 {
   /**
-   * Tunnel Queue.
+   * Double linked list, FIFO style
    */
-  struct CadetTunnelQueue *tq;
+  struct CadetReliableMessage *next;
 
   /**
-   * Message type (DATA/DATA_ACK)
+   * Double linked list, FIFO style
    */
-  uint16_t type;
+  struct CadetReliableMessage *prev;
 
   /**
-   * Message copy (for DATAs, to start retransmission timer)
+   * Which channel is this message in?
    */
-  struct CadetReliableMessage *copy;
+  struct CadetChannel *ch;
 
   /**
-   * Reliability (for DATA_ACKs, to access rel->ack_q)
+   * Entry in the tunnels queue for this message, NULL if it has left
+   * the tunnel.  Used to cancel transmission in case we receive an
+   * ACK in time.
    */
-  struct CadetChannelReliability *rel;
-};
-
+  struct CadetTunnelQueueEntry *qe;
 
-/**
- * Info needed to retry a message in case it gets lost.
- */
-struct CadetReliableMessage
-{
   /**
-   * Double linked list, FIFO style
+   * Data message we are trying to send.
    */
-  struct CadetReliableMessage   *next;
-  struct CadetReliableMessage   *prev;
+  struct GNUNET_CADET_ChannelAppDataMessage *data_message;
 
   /**
-   * Type of message (payload, channel management).
+   * How soon should we retry if we fail to get an ACK?
+   * Messages in the queue are sorted by this value.
    */
-  int16_t                       type;
+  struct GNUNET_TIME_Absolute next_retry;
 
   /**
-   * Tunnel Reliability queue this message is in.
+   * How long do we wait for an ACK after transmission?
+   * Use for the back-off calculation.
    */
-  struct CadetChannelReliability *rel;
+  struct GNUNET_TIME_Relative retry_delay;
 
   /**
-   * ID of the message (ACK needed to free)
+   * Time when we first successfully transmitted the message
+   * (that is, set @e num_transmissions to 1).
    */
-  uint32_t                      mid;
+  struct GNUNET_TIME_Absolute first_transmission_time;
 
   /**
-   * Tunnel Queue.
+   * Identifier of the connection that this message took when it
+   * was first transmitted.  Only useful if @e num_transmissions is 1.
    */
-  struct CadetChannelQueue      *chq;
+  struct GNUNET_CADET_ConnectionTunnelIdentifier connection_taken;
 
   /**
-   * When was this message issued (to calculate ACK delay)
+   * How often was this message transmitted?  #GNUNET_SYSERR if there
+   * was an error transmitting the message, #GNUNET_NO if it was not
+   * yet transmitted ever, otherwise the number of (re) transmissions.
    */
-  struct GNUNET_TIME_Absolute   timestamp;
+  int num_transmissions;
 
-  /* struct GNUNET_CADET_ChannelAppDataMessage with payload */
 };
 
 
 /**
- * Info about the traffic state for a client in a channel.
+ * List of received out-of-order data messages.
  */
-struct CadetChannelReliability
+struct CadetOutOfOrderMessage
 {
   /**
-   * Channel this is about.
+   * Double linked list, FIFO style
    */
-  struct CadetChannel *ch;
+  struct CadetOutOfOrderMessage *next;
 
   /**
-   * DLL of messages sent and not yet ACK'd.
+   * Double linked list, FIFO style
    */
-  struct CadetReliableMessage        *head_sent;
-  struct CadetReliableMessage        *tail_sent;
+  struct CadetOutOfOrderMessage *prev;
 
   /**
-   * DLL of messages received out of order.
+   * ID of the message (messages up to this point needed
+   * before we give this one to the client).
    */
-  struct CadetReliableMessage        *head_recv;
-  struct CadetReliableMessage        *tail_recv;
+  struct ChannelMessageIdentifier mid;
 
   /**
-   * Messages received.
+   * The envelope with the payload of the out-of-order message
    */
-  unsigned int                      n_recv;
+  struct GNUNET_MQ_Envelope *env;
 
-  /**
-   * Next MID to use for outgoing traffic.
-   */
-  uint32_t                          mid_send;
+};
 
-  /**
-   * Next MID expected for incoming traffic.
-   */
-  uint32_t                          mid_recv;
 
+/**
+ * Client endpoint of a `struct CadetChannel`.  A channel may be a
+ * loopback channel, in which case it has two of these endpoints.
+ * Note that flow control also is required in both directions.
+ */
+struct CadetChannelClient
+{
   /**
-   * Handle for queued unique data CREATE, DATA_ACK.
+   * Client handle.  Not by itself sufficient to designate
+   * the client endpoint, as the same client handle may
+   * be used for both the owner and the destination, and
+   * we thus also need the channel ID to identify the client.
    */
-  struct CadetChannelQueue           *uniq;
+  struct CadetClient *c;
 
   /**
-   * Can we send data to the client?
+   * Head of DLL of messages received out of order or while client was unready.
    */
-  int                               client_ready;
+  struct CadetOutOfOrderMessage *head_recv;
 
   /**
-   * Can the client send data to us?
+   * Tail DLL of messages received out of order or while client was unready.
    */
-  int                               client_allowed;
+  struct CadetOutOfOrderMessage *tail_recv;
 
   /**
-   * Task to resend/poll in case no ACK is received.
+   * Local tunnel number for this client.
+   * (if owner >= #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI,
+   *  otherwise < #GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
    */
-  struct GNUNET_SCHEDULER_Task *   retry_task;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
 
   /**
-   * Counter for exponential backoff.
+   * Number of entries currently in @a head_recv DLL.
    */
-  struct GNUNET_TIME_Relative       retry_timer;
+  unsigned int num_recv;
 
   /**
-   * How long does it usually take to get an ACK.
+   * Can we send data to the client?
    */
-  struct GNUNET_TIME_Relative       expected_delay;
+  int client_ready;
+
 };
 
 
@@ -209,41 +253,44 @@ struct CadetChannel
   struct CadetTunnel *t;
 
   /**
-   * Destination port of the channel.
+   * Client owner of the tunnel, if any.
+   * (Used if this channel represends the initiating end of the tunnel.)
    */
-  struct GNUNET_HashCode port;
+  struct CadetChannelClient *owner;
 
   /**
-   * Global channel number ( < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+   * Client destination of the tunnel, if any.
+   * (Used if this channel represents the listening end of the tunnel.)
    */
-  struct GNUNET_CADET_ChannelTunnelNumber gid;
+  struct CadetChannelClient *dest;
 
   /**
-   * Local tunnel number for root (owner) client.
-   * ( >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI or 0 )
+   * Last entry in the tunnel's queue relating to control messages
+   * (#GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN or
+   * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK).  Used to cancel
+   * transmission in case we receive updated information.
    */
-  struct GNUNET_CADET_ClientChannelNumber lid_root;
+  struct CadetTunnelQueueEntry *last_control_qe;
 
   /**
-   * Local tunnel number for local destination clients (incoming number)
-   * ( >= GNUNET_CADET_LOCAL_CHANNEL_ID_SERV or 0).
+   * Head of DLL of messages sent and not yet ACK'd.
    */
-  struct GNUNET_CADET_ClientChannelNumber lid_dest;
+  struct CadetReliableMessage *head_sent;
 
   /**
-   * Channel state.
+   * Tail of DLL of messages sent and not yet ACK'd.
    */
-  enum CadetChannelState state;
+  struct CadetReliableMessage *tail_sent;
 
   /**
-   * Is the tunnel bufferless (minimum latency)?
+   * Task to resend/poll in case no ACK is received.
    */
-  int nobuffer;
+  struct GNUNET_SCHEDULER_Task *retry_control_task;
 
   /**
-   * Is the tunnel reliable?
+   * Task to resend/poll in case no ACK is received.
    */
-  int reliable;
+  struct GNUNET_SCHEDULER_Task *retry_data_task;
 
   /**
    * Last time the channel was used
@@ -251,21 +298,29 @@ struct CadetChannel
   struct GNUNET_TIME_Absolute timestamp;
 
   /**
-   * Client owner of the tunnel, if any
+   * Destination port of the channel.
    */
-  struct CadetClient *root;
+  struct GNUNET_HashCode port;
 
   /**
-   * Client destination of the tunnel, if any.
+   * Counter for exponential backoff.
    */
-  struct CadetClient *dest;
+  struct GNUNET_TIME_Relative retry_time;
 
   /**
-   * Flag to signal the destruction of the channel.
-   * If this is set to #GNUNET_YES the channel will be destroyed
-   * when the queue is empty.
+   * Bitfield of already-received messages past @e mid_recv.
    */
-  int destroy;
+  uint64_t mid_futures;
+
+  /**
+   * Next MID expected for incoming traffic.
+   */
+  struct ChannelMessageIdentifier mid_recv;
+
+  /**
+   * Next MID to use for outgoing traffic.
+   */
+  struct ChannelMessageIdentifier mid_send;
 
   /**
    * Total (reliable) messages pending ACK for this channel.
@@ -273,2290 +328,1710 @@ struct CadetChannel
   unsigned int pending_messages;
 
   /**
-   * Reliability data.
-   * Only present (non-NULL) at the owner of a tunnel.
+   * Maximum (reliable) messages pending ACK for this channel
+   * before we throttle the client.
    */
-  struct CadetChannelReliability *root_rel;
+  unsigned int max_pending_messages;
 
   /**
-   * Reliability data.
-   * Only present (non-NULL) at the destination of a tunnel.
+   * Number identifying this channel in its tunnel.
    */
-  struct CadetChannelReliability *dest_rel;
+  struct GNUNET_CADET_ChannelTunnelNumber ctn;
 
-};
+  /**
+   * Channel state.
+   */
+  enum CadetChannelState state;
 
+  /**
+   * Count how many ACKs we skipped, used to prevent long
+   * sequences of ACK skipping.
+   */
+  unsigned int skip_ack_series;
 
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
+  /**
+   * Is the tunnel bufferless (minimum latency)?
+   */
+  int nobuffer;
 
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
+  /**
+   * Is the tunnel reliable?
+   */
+  int reliable;
 
-/**
- * Local peer own ID (memory efficient handle).
- */
-extern GNUNET_PEER_Id myid;
+  /**
+   * Is the tunnel out-of-order?
+   */
+  int out_of_order;
 
+  /**
+   * Is this channel a loopback channel, where the destination is us again?
+   */
+  int is_loopback;
 
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
+  /**
+   * Flag to signal the destruction of the channel.  If this is set to
+   * #GNUNET_YES the channel will be destroyed once the queue is
+   * empty.
+   */
+  int destroy;
 
+};
 
-/**
- * Destroy a reliable message after it has been acknowledged, either by
- * direct mid ACK or bitfield. Updates the appropriate data structures and
- * timers and frees all memory.
- *
- * @param copy Message that is no longer needed: remote peer got it.
- * @param update_time Is the timing information relevant?
- *                    If this message is ACK in a batch the timing information
- *                    is skewed by the retransmission, count only for the
- *                    retransmitted message.
- *
- * @return #GNUNET_YES if channel was destroyed as a result of the call,
- *         #GNUNET_NO otherwise.
- */
-static int
-rel_message_free (struct CadetReliableMessage *copy, int update_time);
 
 /**
- * send a channel create message.
+ * Get the static string for identification of the channel.
  *
- * @param ch Channel for which to send.
- */
-static void
-send_create (struct CadetChannel *ch);
-
-/**
- * Confirm we got a channel create, FWD ack.
+ * @param ch Channel.
  *
- * @param ch The channel to confirm.
- * @param fwd Should we send a FWD ACK? (going dest->root)
+ * @return Static string with the channel IDs.
  */
-static void
-send_ack (struct CadetChannel *ch, int fwd);
-
+const char *
+GCCH_2s (const struct CadetChannel *ch)
+{
+  static char buf[128];
+
+  GNUNET_snprintf (buf,
+                   sizeof (buf),
+                   "Channel %s:%s ctn:%X(%X/%X)",
+                   (GNUNET_YES == ch->is_loopback)
+                   ? "loopback"
+                   : GNUNET_i2s (GCP_get_id (GCT_get_destination (ch->t))),
+                   GNUNET_h2s (&ch->port),
+                   ch->ctn,
+                   (NULL == ch->owner) ? 0 : ntohl (ch->owner->ccn.channel_of_client),
+                   (NULL == ch->dest) ? 0 : ntohl (ch->dest->ccn.channel_of_client));
+  return buf;
+}
 
 
 /**
- * Test if the channel is loopback: both root and dest are on the local peer.
+ * Get the channel's public ID.
  *
- * @param ch Channel to test.
+ * @param ch Channel.
  *
- * @return #GNUNET_YES if channel is loopback, #GNUNET_NO otherwise.
+ * @return ID used to identify the channel with the remote peer.
  */
-static int
-is_loopback (const struct CadetChannel *ch)
+struct GNUNET_CADET_ChannelTunnelNumber
+GCCH_get_id (const struct CadetChannel *ch)
 {
-  if (NULL != ch->t)
-    return GCT_is_loopback (ch->t);
-
-  return (NULL != ch->root && NULL != ch->dest);
+  return ch->ctn;
 }
 
 
 /**
- * Save a copy of the data message for later retransmission.
+ * Release memory associated with @a ccc
  *
- * @param msg Message to copy.
- * @param mid Message ID.
- * @param rel Reliability data for retransmission.
+ * @param ccc data structure to clean up
  */
-static struct CadetReliableMessage *
-copy_message (const struct GNUNET_CADET_ChannelAppDataMessage *msg, uint32_t mid,
-              struct CadetChannelReliability *rel)
+static void
+free_channel_client (struct CadetChannelClient *ccc)
 {
-  struct CadetReliableMessage *copy;
-  uint16_t size;
+  struct CadetOutOfOrderMessage *com;
 
-  size = ntohs (msg->header.size);
-  copy = GNUNET_malloc (sizeof (*copy) + size);
-  copy->mid = mid;
-  copy->rel = rel;
-  copy->type = GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA;
-  GNUNET_memcpy (&copy[1], msg, size);
-
-  return copy;
+  while (NULL != (com = ccc->head_recv))
+  {
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                                 ccc->tail_recv,
+                                 com);
+    ccc->num_recv--;
+    GNUNET_MQ_discard (com->env);
+    GNUNET_free (com);
+  }
+  GNUNET_free (ccc);
 }
 
+
 /**
- * We have received a message out of order, or the client is not ready.
- * Buffer it until we receive an ACK from the client or the missing
- * message from the channel.
+ * Destroy the given channel.
  *
- * @param msg Message to buffer (MUST be of type CADET_DATA).
- * @param rel Reliability data to the corresponding direction.
+ * @param ch channel to destroy
  */
 static void
-add_buffered_data (const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-                   struct CadetChannelReliability *rel)
+channel_destroy (struct CadetChannel *ch)
 {
-  struct CadetReliableMessage *copy;
-  struct CadetReliableMessage *prev;
-  uint32_t mid;
-
-  mid = ntohl (msg->mid);
+  struct CadetReliableMessage *crm;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "add_buffered_data MID %u (%u)\n",
-       mid, rel->n_recv);
-
-  rel->n_recv++;
-
-  // FIXME do something better than O(n), although n < 64...
-  // FIXME start from the end (most messages are the latest ones)
-  for (prev = rel->head_recv; NULL != prev; prev = prev->next)
+  while (NULL != (crm = ch->head_sent))
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " prev %u\n", prev->mid);
-    if (prev->mid == mid)
+    GNUNET_assert (ch == crm->ch);
+    if (NULL != crm->qe)
     {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " already there!\n");
-      rel->n_recv--;
-      return;
-    }
-    else if (GC_is_pid_bigger (prev->mid, mid))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " bingo!\n");
-      copy = copy_message (msg, mid, rel);
-      GNUNET_CONTAINER_DLL_insert_before (rel->head_recv, rel->tail_recv,
-                                          prev, copy);
-      return;
+      GCT_send_cancel (crm->qe);
+      crm->qe = NULL;
     }
+    GNUNET_CONTAINER_DLL_remove (ch->head_sent,
+                                 ch->tail_sent,
+                                 crm);
+    GNUNET_free (crm->data_message);
+    GNUNET_free (crm);
+  }
+  if (NULL != ch->owner)
+  {
+    free_channel_client (ch->owner);
+    ch->owner = NULL;
   }
-  copy = copy_message (msg, mid, rel);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " insert at tail! (now: %u)\n", rel->n_recv);
-  GNUNET_CONTAINER_DLL_insert_tail (rel->head_recv, rel->tail_recv, copy);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "add_buffered_data END\n");
+  if (NULL != ch->dest)
+  {
+    free_channel_client (ch->dest);
+    ch->dest = NULL;
+  }
+  if (NULL != ch->last_control_qe)
+  {
+    GCT_send_cancel (ch->last_control_qe);
+    ch->last_control_qe = NULL;
+  }
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
+  if (NULL != ch->retry_control_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+    ch->retry_control_task = NULL;
+  }
+  if (GNUNET_NO == ch->is_loopback)
+  {
+    GCT_remove_channel (ch->t,
+                        ch,
+                        ch->ctn);
+    ch->t = NULL;
+  }
+  GNUNET_free (ch);
 }
 
 
 /**
- * Add a destination client to a channel, initializing all data structures
- * in the channel and the client.
+ * Send a channel create message.
  *
- * @param ch Channel to which add the destination.
- * @param c Client which to add to the channel.
+ * @param cls Channel for which to send.
  */
 static void
-add_destination (struct CadetChannel *ch, struct CadetClient *c)
-{
-  if (NULL != ch->dest)
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* Assign local id as destination */
-  ch->lid_dest = GML_get_next_ccn (c);
-
-  /* Store in client's hashmap */
-  GML_channel_add (c, ch->lid_dest, ch);
-
-  GNUNET_break (NULL == ch->dest_rel);
-  ch->dest_rel = GNUNET_new (struct CadetChannelReliability);
-  ch->dest_rel->ch = ch;
-  ch->dest_rel->expected_delay.rel_value_us = 0;
-  ch->dest_rel->retry_timer = CADET_RETRANSMIT_TIME;
-
-  ch->dest = c;
-}
+send_channel_open (void *cls);
 
 
 /**
- * Set options in a channel, extracted from a bit flag field.
+ * Function called once the tunnel confirms that we sent the
+ * create message.  Delays for a bit until we retry.
  *
- * @param ch Channel to set options to.
- * @param options Bit array in host byte order.
+ * @param cls our `struct CadetChannel`.
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
  */
 static void
-channel_set_options (struct CadetChannel *ch, uint32_t options)
+channel_open_sent_cb (void *cls,
+                      const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  ch->nobuffer = (options & GNUNET_CADET_OPTION_NOBUFFER) != 0 ?
-  GNUNET_YES : GNUNET_NO;
-  ch->reliable = (options & GNUNET_CADET_OPTION_RELIABLE) != 0 ?
-  GNUNET_YES : GNUNET_NO;
+  struct CadetChannel *ch = cls;
+
+  GNUNET_assert (NULL != ch->last_control_qe);
+  ch->last_control_qe = NULL;
+  ch->retry_time = GNUNET_TIME_STD_BACKOFF (ch->retry_time);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sent CADET_CHANNEL_OPEN on %s, retrying in %s\n",
+       GCCH_2s (ch),
+       GNUNET_STRINGS_relative_time_to_string (ch->retry_time,
+                                               GNUNET_YES));
+  ch->retry_control_task
+    = GNUNET_SCHEDULER_add_delayed (ch->retry_time,
+                                    &send_channel_open,
+                                    ch);
 }
 
 
 /**
- * Get a bit flag field with the options of a channel.
+ * Send a channel open message.
  *
- * @param ch Channel to get options from.
- *
- * @return Bit array in host byte order.
+ * @param cls Channel for which to send.
  */
-static uint32_t
-channel_get_options (struct CadetChannel *ch)
+static void
+send_channel_open (void *cls)
 {
+  struct CadetChannel *ch = cls;
+  struct GNUNET_CADET_ChannelOpenMessage msgcc;
   uint32_t options;
 
+  ch->retry_control_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CHANNEL_OPEN message for %s\n",
+       GCCH_2s (ch));
   options = 0;
   if (ch->nobuffer)
     options |= GNUNET_CADET_OPTION_NOBUFFER;
   if (ch->reliable)
     options |= GNUNET_CADET_OPTION_RELIABLE;
-
-  return options;
+  if (ch->out_of_order)
+    options |= GNUNET_CADET_OPTION_OUT_OF_ORDER;
+  msgcc.header.size = htons (sizeof (msgcc));
+  msgcc.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN);
+  msgcc.opt = htonl (options);
+  msgcc.port = ch->port;
+  msgcc.ctn = ch->ctn;
+  ch->state = CADET_CHANNEL_OPEN_SENT;
+  if (NULL != ch->last_control_qe)
+    GCT_send_cancel (ch->last_control_qe);
+  ch->last_control_qe = GCT_send (ch->t,
+                                  &msgcc.header,
+                                  &channel_open_sent_cb,
+                                  ch);
+  GNUNET_assert (NULL == ch->retry_control_task);
 }
 
 
 /**
- * Notify a client that the channel is no longer valid.
- *
- * @param ch Channel that is destroyed.
- * @param local_only Should we avoid sending it to other peers?
+ * Function called once and only once after a channel was bound
+ * to its tunnel via #GCT_add_channel() is ready for transmission.
+ * Note that this is only the case for channels that this peer
+ * initiates, as for incoming channels we assume that they are
+ * ready for transmission immediately upon receiving the open
+ * message.  Used to bootstrap the #GCT_send() process.
+ *
+ * @param ch the channel for which the tunnel is now ready
  */
-static void
-send_destroy (struct CadetChannel *ch, int local_only)
+void
+GCCH_tunnel_up (struct CadetChannel *ch)
 {
-  struct GNUNET_CADET_ChannelManageMessage msg;
-
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY);
-  msg.header.size = htons (sizeof (msg));
-  msg.ctn = ch->gid;
-
-  /* If root is not NULL, notify.
-   * If it's NULL, check lid_root. When a local destroy comes in, root
-   * is set to NULL but lid_root is left untouched. In this case, do nothing,
-   * the client is the one who requested the channel to be destroyed.
-   */
-  if (NULL != ch->root)
-    GML_send_channel_destroy (ch->root, ch->lid_root);
-  else if (0 == ch->lid_root.channel_of_client && GNUNET_NO == local_only)
-    GCCH_send_prebuilt_message (&msg.header, ch, GNUNET_NO, NULL);
-
-  if (NULL != ch->dest)
-    GML_send_channel_destroy (ch->dest, ch->lid_dest);
-  else if (0 == ch->lid_dest.channel_of_client && GNUNET_NO == local_only)
-    GCCH_send_prebuilt_message (&msg.header, ch, GNUNET_YES, NULL);
+  GNUNET_assert (NULL == ch->retry_control_task);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Tunnel up, sending CHANNEL_OPEN on %s now\n",
+       GCCH_2s (ch));
+  ch->retry_control_task
+    = GNUNET_SCHEDULER_add_now (&send_channel_open,
+                                ch);
 }
 
 
 /**
- * Notify the destination client that a new incoming channel was created.
+ * Create a new channel.
  *
- * @param ch Channel that was created.
+ * @param owner local client owning the channel
+ * @param ccn local number of this channel at the @a owner
+ * @param destination peer to which we should build the channel
+ * @param port desired port at @a destination
+ * @param options options for the channel
+ * @return handle to the new channel
  */
-static void
-send_client_create (struct CadetChannel *ch)
+struct CadetChannel *
+GCCH_channel_local_new (struct CadetClient *owner,
+                        struct GNUNET_CADET_ClientChannelNumber ccn,
+                        struct CadetPeer *destination,
+                        const struct GNUNET_HashCode *port,
+                        uint32_t options)
 {
-  uint32_t opt;
-
-  if (NULL == ch->dest)
-    return;
-
-  opt = 0;
-  opt |= GNUNET_YES == ch->reliable ? GNUNET_CADET_OPTION_RELIABLE : 0;
-  opt |= GNUNET_YES == ch->nobuffer ? GNUNET_CADET_OPTION_NOBUFFER : 0;
-  GML_send_channel_create (ch->dest,
-                           ch->lid_dest,
-                           &ch->port,
-                           opt,
-                           GCT_get_destination (ch->t));
-
-}
+  struct CadetChannel *ch;
+  struct CadetChannelClient *ccco;
 
+  ccco = GNUNET_new (struct CadetChannelClient);
+  ccco->c = owner;
+  ccco->ccn = ccn;
+  ccco->client_ready = GNUNET_YES;
 
-/**
- * Send data to a client.
- *
- * If the client is ready, send directly, otherwise buffer while listening
- * for a local ACK.
- *
- * @param ch Channel
- * @param msg Message.
- * @param fwd Is this a fwd (root->dest) message?
- */
-static void
-send_client_data (struct CadetChannel *ch,
-                  const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-                  int fwd)
-{
-  if (fwd)
-  {
-    if (ch->dest_rel->client_ready)
+  ch = GNUNET_new (struct CadetChannel);
+  ch->mid_recv.mid = htonl (1); /* The OPEN_ACK counts as message 0! */
+  ch->nobuffer = (0 != (options & GNUNET_CADET_OPTION_NOBUFFER));
+  ch->reliable = (0 != (options & GNUNET_CADET_OPTION_RELIABLE));
+  ch->out_of_order = (0 != (options & GNUNET_CADET_OPTION_OUT_OF_ORDER));
+  ch->max_pending_messages = (ch->nobuffer) ? 1 : 4; /* FIXME: 4!? Do not hardcode! */
+  ch->owner = ccco;
+  ch->port = *port;
+  if (0 == memcmp (&my_full_id,
+                   GCP_get_id (destination),
+                   sizeof (struct GNUNET_PeerIdentity)))
+  {
+    struct CadetClient *c;
+
+    ch->is_loopback = GNUNET_YES;
+    c = GNUNET_CONTAINER_multihashmap_get (open_ports,
+                                           port);
+    if (NULL == c)
     {
-      GML_send_data (ch->dest, msg, ch->lid_dest);
-      ch->dest_rel->client_ready = GNUNET_NO;
-      ch->dest_rel->mid_recv++;
+      /* port closed, wait for it to possibly open */
+      ch->state = CADET_CHANNEL_LOOSE;
+      (void) GNUNET_CONTAINER_multihashmap_put (loose_channels,
+                                                port,
+                                                ch,
+                                                GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Created loose incoming loopback channel to port %s\n",
+           GNUNET_h2s (&ch->port));
     }
     else
-      add_buffered_data (msg, ch->dest_rel);
+    {
+      GCCH_bind (ch,
+                 c);
+    }
   }
   else
   {
-    if (ch->root_rel->client_ready)
-    {
-      GML_send_data (ch->root, msg, ch->lid_root);
-      ch->root_rel->client_ready = GNUNET_NO;
-      ch->root_rel->mid_recv++;
-    }
-    else
-      add_buffered_data (msg, ch->root_rel);
+    ch->t = GCP_get_tunnel (destination,
+                            GNUNET_YES);
+    ch->retry_time = CADET_INITIAL_RETRANSMIT_TIME;
+    ch->ctn = GCT_add_channel (ch->t,
+                               ch);
   }
+  GNUNET_STATISTICS_update (stats,
+                            "# channels",
+                            1,
+                            GNUNET_NO);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Created channel to port %s at peer %s for %s using %s\n",
+       GNUNET_h2s (port),
+       GCP_2s (destination),
+       GSC_2s (owner),
+       (GNUNET_YES == ch->is_loopback) ? "loopback" : GCT_2s (ch->t));
+  return ch;
 }
 
 
 /**
- * Send a buffered message to the client, for in order delivery or
- * as result of client ACK.
+ * We had an incoming channel to a port that is closed.
+ * It has not been opened for a while, drop it.
  *
- * @param ch Channel on which to empty the message buffer.
- * @param c Client to send to.
- * @param fwd Is this to send FWD data?.
+ * @param cls the channel to drop
  */
 static void
-send_client_buffered_data (struct CadetChannel *ch,
-                           struct CadetClient *c,
-                           int fwd)
+timeout_closed_cb (void *cls)
 {
-  struct CadetReliableMessage *copy;
-  struct CadetChannelReliability *rel;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "send_buffered_data\n");
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-  if (GNUNET_NO == rel->client_ready)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "client not ready\n");
-    return;
-  }
+  struct CadetChannel *ch = cls;
 
-  copy = rel->head_recv;
-  /* We never buffer channel management messages */
-  if (NULL != copy)
-  {
-    if (copy->mid == rel->mid_recv || GNUNET_NO == ch->reliable)
-    {
-      struct GNUNET_CADET_ChannelAppDataMessage *msg = (struct GNUNET_CADET_ChannelAppDataMessage *) &copy[1];
-
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " have %u! now expecting %u\n",
-           copy->mid, rel->mid_recv + 1);
-      send_client_data (ch, msg, fwd);
-      rel->n_recv--;
-      GNUNET_CONTAINER_DLL_remove (rel->head_recv, rel->tail_recv, copy);
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " free copy recv MID %u (%p), %u left\n",
-           copy->mid, copy, rel->n_recv);
-      GNUNET_free (copy);
-      GCCH_send_data_ack (ch, fwd);
-    }
-    else
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " reliable && don't have %u, next is %u\n",
-           rel->mid_recv, copy->mid);
-      if (GNUNET_YES == ch->destroy)
-      {
-        /* We don't have the next data piece and the remote peer has closed the
-         * channel. We won't receive it anymore, so just destroy the channel.
-         * FIXME: wait some time to allow other connections to
-         *        deliver missing messages
-         */
-        send_destroy (ch, GNUNET_YES);
-        GCCH_destroy (ch);
-      }
-    }
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "send_buffered_data END\n");
+  ch->retry_control_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Closing incoming channel to port %s from peer %s due to timeout\n",
+       GNUNET_h2s (&ch->port),
+       GCP_2s (GCT_get_destination (ch->t)));
+  channel_destroy (ch);
 }
 
 
 /**
- * Allow a client to send more data.
- *
- * In case the client was already allowed to send data, do nothing.
+ * Create a new channel based on a request coming in over the network.
  *
- * @param ch Channel.
- * @param fwd Is this a FWD ACK? (FWD ACKs are sent to root)
+ * @param t tunnel to the remote peer
+ * @param ctn identifier of this channel in the tunnel
+ * @param port desired local port
+ * @param options options for the channel
+ * @return handle to the new channel
  */
-static void
-send_client_ack (struct CadetChannel *ch, int fwd)
+struct CadetChannel *
+GCCH_channel_incoming_new (struct CadetTunnel *t,
+                           struct GNUNET_CADET_ChannelTunnelNumber ctn,
+                           const struct GNUNET_HashCode *port,
+                           uint32_t options)
 {
-  struct CadetChannelReliability *rel = fwd ? ch->root_rel : ch->dest_rel;
-  struct CadetClient *c = fwd ? ch->root : ch->dest;
+  struct CadetChannel *ch;
+  struct CadetClient *c;
+
+  ch = GNUNET_new (struct CadetChannel);
+  ch->port = *port;
+  ch->t = t;
+  ch->ctn = ctn;
+  ch->retry_time = CADET_INITIAL_RETRANSMIT_TIME;
+  ch->nobuffer = (0 != (options & GNUNET_CADET_OPTION_NOBUFFER));
+  ch->reliable = (0 != (options & GNUNET_CADET_OPTION_RELIABLE));
+  ch->out_of_order = (0 != (options & GNUNET_CADET_OPTION_OUT_OF_ORDER));
+  ch->max_pending_messages = (ch->nobuffer) ? 1 : 4; /* FIXME: 4!? Do not hardcode! */
+  GNUNET_STATISTICS_update (stats,
+                            "# channels",
+                            1,
+                            GNUNET_NO);
 
+  c = GNUNET_CONTAINER_multihashmap_get (open_ports,
+                                         port);
   if (NULL == c)
   {
-    GNUNET_break (GNUNET_NO != ch->destroy);
-    return;
+    /* port closed, wait for it to possibly open */
+    ch->state = CADET_CHANNEL_LOOSE;
+    (void) GNUNET_CONTAINER_multihashmap_put (loose_channels,
+                                              port,
+                                              ch,
+                                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+    GNUNET_assert (NULL == ch->retry_control_task);
+    ch->retry_control_task
+      = GNUNET_SCHEDULER_add_delayed (TIMEOUT_CLOSED_PORT,
+                                      &timeout_closed_cb,
+                                      ch);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Created loose incoming channel to port %s from peer %s\n",
+         GNUNET_h2s (&ch->port),
+         GCP_2s (GCT_get_destination (ch->t)));
   }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "  sending %s ack to client on channel %s\n",
-       GC_f2s (fwd), GCCH_2s (ch));
-
-  if (NULL == rel)
+  else
   {
-    GNUNET_break (0);
-    return;
+    GCCH_bind (ch,
+               c);
   }
-
-  if (GNUNET_YES == rel->client_allowed)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  already allowed\n");
-    return;
-  }
-  rel->client_allowed = GNUNET_YES;
-
-  GML_send_ack (c, fwd ? ch->lid_root : ch->lid_dest);
+  GNUNET_STATISTICS_update (stats,
+                            "# channels",
+                            1,
+                            GNUNET_NO);
+  return ch;
 }
 
 
 /**
- * Notify the root that the destination rejected the channel.
+ * Function called once the tunnel confirms that we sent the
+ * ACK message.  Just remembers it was sent, we do not expect
+ * ACKs for ACKs ;-).
  *
- * @param ch Rejected channel.
+ * @param cls our `struct CadetChannel`.
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
  */
 static void
-send_client_nack (struct CadetChannel *ch)
+send_ack_cb (void *cls,
+             const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  if (NULL == ch->root)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  GML_send_channel_nack (ch->root, ch->lid_root);
+  struct CadetChannel *ch = cls;
+
+  GNUNET_assert (NULL != ch->last_control_qe);
+  ch->last_control_qe = NULL;
 }
 
 
 /**
- * We haven't received an ACK after a certain time: restransmit the message.
+ * Compute and send the current #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK to the other peer.
  *
- * @param cls Closure (CadetChannelReliability with the message to restransmit)
+ * @param ch channel to send the #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK for
  */
 static void
-channel_retransmit_message (void *cls)
+send_channel_data_ack (struct CadetChannel *ch)
 {
-  struct CadetChannelReliability *rel = cls;
-  struct CadetReliableMessage *copy;
-  struct CadetChannel *ch;
-  struct GNUNET_CADET_ChannelAppDataMessage *payload;
-  int fwd;
-
-  rel->retry_task = NULL;
-  ch = rel->ch;
-  copy = rel->head_sent;
-  if (NULL == copy)
-  {
-    GNUNET_break (0); // FIXME tripped in rps testcase
-    return;
-  }
-
-  payload = (struct GNUNET_CADET_ChannelAppDataMessage *) &copy[1];
-  fwd = (rel == ch->root_rel);
-
-  /* Message not found in the queue that we are going to use. */
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "RETRANSMIT MID %u\n", copy->mid);
+  struct GNUNET_CADET_ChannelDataAckMessage msg;
 
-  GCCH_send_prebuilt_message (&payload->header, ch, fwd, copy);
-  GNUNET_STATISTICS_update (stats, "# data retransmitted", 1, GNUNET_NO);
+  if (GNUNET_NO == ch->reliable)
+    return; /* no ACKs */
+  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK);
+  msg.header.size = htons (sizeof (msg));
+  msg.ctn = ch->ctn;
+  msg.mid.mid = htonl (ntohl (ch->mid_recv.mid));
+  msg.futures = GNUNET_htonll (ch->mid_futures);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending DATA_ACK %u:%llX via %s\n",
+       (unsigned int) ntohl (msg.mid.mid),
+       (unsigned long long) ch->mid_futures,
+       GCCH_2s (ch));
+  if (NULL != ch->last_control_qe)
+    GCT_send_cancel (ch->last_control_qe);
+  ch->last_control_qe = GCT_send (ch->t,
+                                  &msg.header,
+                                  &send_ack_cb,
+                                  ch);
 }
 
 
 /**
- * We haven't received an Channel ACK after a certain time: resend the CREATE.
+ * Send our initial #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK to the client confirming that the
+ * connection is up.
  *
- * @param cls Closure (CadetChannelReliability of the channel to recreate)
+ * @param cls the `struct CadetChannel`
  */
 static void
-channel_recreate (void *cls)
+send_open_ack (void *cls)
 {
-  struct CadetChannelReliability *rel = cls;
-
-  rel->retry_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "RE-CREATE\n");
-  GNUNET_STATISTICS_update (stats,
-                            "# data retransmitted", 1, GNUNET_NO);
+  struct CadetChannel *ch = cls;
+  struct GNUNET_CADET_ChannelManageMessage msg;
 
-  if (rel == rel->ch->root_rel)
-  {
-    send_create (rel->ch);
-  }
-  else if (rel == rel->ch->dest_rel)
-  {
-    send_ack (rel->ch, GNUNET_YES);
-  }
-  else
-  {
-    GNUNET_break (0);
-  }
+  ch->retry_control_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CHANNEL_OPEN_ACK on %s\n",
+       GCCH_2s (ch));
+  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK);
+  msg.header.size = htons (sizeof (msg));
+  msg.reserved = htonl (0);
+  msg.ctn = ch->ctn;
+  if (NULL != ch->last_control_qe)
+    GCT_send_cancel (ch->last_control_qe);
+  ch->last_control_qe = GCT_send (ch->t,
+                                  &msg.header,
+                                  &send_ack_cb,
+                                  ch);
 }
 
 
 /**
- * Message has been sent: start retransmission timer.
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN message again for
+ * this channel.  If the binding was successful, (re)transmit the
+ * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
  *
- * @param cls Closure (queue structure).
- * @param t Tunnel.
- * @param q Queue handler (no longer valid).
- * @param type Type of message.
- * @param size Size of the message.
+ * @param ch channel that got the duplicate open
+ * @param cti identifier of the connection that delivered the message
  */
-static void
-ch_message_sent (void *cls,
-                 struct CadetTunnel *t,
-                 struct CadetTunnelQueue *q,
-                 uint16_t type, size_t size)
+void
+GCCH_handle_duplicate_open (struct CadetChannel *ch,
+                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti)
 {
-  struct CadetChannelQueue *chq = cls;
-  struct CadetReliableMessage *copy = chq->copy;
-  struct CadetChannelReliability *rel;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "channel_message_sent callback %s\n",
-       GC_m2s (chq->type));
-
-  switch (chq->type)
+  if (NULL == ch->dest)
   {
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "data MID %u sent\n", copy->mid);
-      GNUNET_assert (chq == copy->chq);
-      copy->timestamp = GNUNET_TIME_absolute_get ();
-      rel = copy->rel;
-      if (NULL == rel->retry_task)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  scheduling retry in %d * %s\n",
-             CADET_RETRANSMIT_MARGIN,
-             GNUNET_STRINGS_relative_time_to_string (rel->expected_delay,
-                                                     GNUNET_YES));
-        if (0 != rel->expected_delay.rel_value_us)
-        {
-          rel->retry_timer =
-              GNUNET_TIME_relative_saturating_multiply (rel->expected_delay,
-                                                        CADET_RETRANSMIT_MARGIN);
-        }
-        else
-        {
-          rel->retry_timer = CADET_RETRANSMIT_TIME;
-        }
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  using delay %s\n",
-             GNUNET_STRINGS_relative_time_to_string (rel->retry_timer,
-                                                     GNUNET_NO));
-        rel->retry_task =
-            GNUNET_SCHEDULER_add_delayed (rel->retry_timer,
-                                          &channel_retransmit_message, rel);
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "retry running %p\n", rel->retry_task);
-      }
-      copy->chq = NULL;
-      break;
-
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "sent %s\n", GC_m2s (chq->type));
-      rel = chq->rel;
-      GNUNET_assert (rel->uniq == chq);
-      rel->uniq = NULL;
-
-      if (CADET_CHANNEL_READY != rel->ch->state
-          && GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK != type
-          && GNUNET_NO == rel->ch->destroy)
-      {
-        GNUNET_assert (NULL == rel->retry_task);
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "STD BACKOFF %s\n",
-             GNUNET_STRINGS_relative_time_to_string (rel->retry_timer,
-                                                     GNUNET_NO));
-        rel->retry_timer = GNUNET_TIME_STD_BACKOFF (rel->retry_timer);
-        rel->retry_task = GNUNET_SCHEDULER_add_delayed (rel->retry_timer,
-                                                        &channel_recreate, rel);
-      }
-      break;
-
-    default:
-      GNUNET_break (0);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring duplicate CHANNEL_OPEN on %s: port is closed\n",
+         GCCH_2s (ch));
+    return;
   }
-
-  GNUNET_free (chq);
+  if (NULL != ch->retry_control_task)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring duplicate CHANNEL_OPEN on %s: control message is pending\n",
+         GCCH_2s (ch));
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Retransmitting CHANNEL_OPEN_ACK on %s\n",
+       GCCH_2s (ch));
+  ch->retry_control_task
+    = GNUNET_SCHEDULER_add_now (&send_open_ack,
+                                ch);
 }
 
 
 /**
- * send a channel create message.
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK to the client to solicit more messages.
  *
- * @param ch Channel for which to send.
+ * @param ch channel the ack is for
+ * @param to_owner #GNUNET_YES to send to owner,
+ *                 #GNUNET_NO to send to dest
  */
 static void
-send_create (struct CadetChannel *ch)
+send_ack_to_client (struct CadetChannel *ch,
+                    int to_owner)
 {
-  struct GNUNET_CADET_ChannelOpenMessage msgcc;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalAck *ack;
+  struct CadetChannelClient *ccc;
 
-  msgcc.header.size = htons (sizeof (msgcc));
-  msgcc.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN);
-  msgcc.ctn = ch->gid;
-  msgcc.port = ch->port;
-  msgcc.opt = htonl (channel_get_options (ch));
-
-  GCCH_send_prebuilt_message (&msgcc.header, ch, GNUNET_YES, NULL);
+  ccc = (GNUNET_YES == to_owner) ? ch->owner : ch->dest;
+  if (NULL == ccc)
+  {
+    /* This can happen if we are just getting ACKs after
+       our local client already disconnected. */
+    GNUNET_assert (GNUNET_YES == ch->destroy);
+    return;
+  }
+  env = GNUNET_MQ_msg (ack,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
+  ack->ccn = ccc->ccn;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CADET_LOCAL_ACK to %s (%s) at ccn %X (%u/%u pending)\n",
+       GSC_2s (ccc->c),
+       (GNUNET_YES == to_owner) ? "owner" : "dest",
+       ntohl (ack->ccn.channel_of_client),
+       ch->pending_messages,
+       ch->max_pending_messages);
+  GSC_send_to_client (ccc->c,
+                      env);
 }
 
 
 /**
- * Confirm we got a channel create or FWD ack.
+ * A client is bound to the port that we have a channel
+ * open to.  Send the acknowledgement for the connection
+ * request and establish the link with the client.
  *
- * @param ch The channel to confirm.
- * @param fwd Should we send a FWD ACK? (going dest->root)
+ * @param ch open incoming channel
+ * @param c client listening on the respective port
  */
-static void
-send_ack (struct CadetChannel *ch, int fwd)
+void
+GCCH_bind (struct CadetChannel *ch,
+           struct CadetClient *c)
 {
-  struct GNUNET_CADET_ChannelManageMessage msg;
+  uint32_t options;
+  struct CadetChannelClient *cccd;
 
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "  sending channel %s ack for channel %s\n",
-       GC_f2s (fwd), GCCH_2s (ch));
-
-  msg.ctn =ch->gid;
-  GCCH_send_prebuilt_message (&msg.header, ch, !fwd, NULL);
+       "Binding %s from %s to port %s of %s\n",
+       GCCH_2s (ch),
+       GCT_2s (ch->t),
+       GNUNET_h2s (&ch->port),
+       GSC_2s (c));
+  if (NULL != ch->retry_control_task)
+  {
+    /* there might be a timeout task here */
+    GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+    ch->retry_control_task = NULL;
+  }
+  options = 0;
+  if (ch->nobuffer)
+    options |= GNUNET_CADET_OPTION_NOBUFFER;
+  if (ch->reliable)
+    options |= GNUNET_CADET_OPTION_RELIABLE;
+  if (ch->out_of_order)
+    options |= GNUNET_CADET_OPTION_OUT_OF_ORDER;
+  cccd = GNUNET_new (struct CadetChannelClient);
+  GNUNET_assert (NULL == ch->dest);
+  ch->dest = cccd;
+  cccd->c = c;
+  cccd->client_ready = GNUNET_YES;
+  cccd->ccn = GSC_bind (c,
+                        ch,
+                        (GNUNET_YES == ch->is_loopback)
+                        ? GCP_get (&my_full_id,
+                                   GNUNET_YES)
+                        : GCT_get_destination (ch->t),
+                        &ch->port,
+                        options);
+  GNUNET_assert (ntohl (cccd->ccn.channel_of_client) <
+                 GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
+  ch->mid_recv.mid = htonl (1); /* The OPEN counts as message 0! */
+  if (GNUNET_YES == ch->is_loopback)
+  {
+    ch->state = CADET_CHANNEL_OPEN_SENT;
+    GCCH_handle_channel_open_ack (ch,
+                                  NULL);
+  }
+  else
+  {
+    /* notify other peer that we accepted the connection */
+    ch->state = CADET_CHANNEL_READY;
+    ch->retry_control_task
+      = GNUNET_SCHEDULER_add_now (&send_open_ack,
+                                  ch);
+  }
+  /* give client it's initial supply of ACKs */
+  GNUNET_assert (ntohl (cccd->ccn.channel_of_client) <
+                 GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
+  for (unsigned int i=0;i<ch->max_pending_messages;i++)
+    send_ack_to_client (ch,
+                        GNUNET_NO);
 }
 
 
 /**
- * Send a message and don't keep any info about it: we won't need to cancel it
- * or resend it.
+ * One of our clients has disconnected, tell the other one that we
+ * are finished. Done asynchronously to avoid concurrent modification
+ * issues if this is the same client.
  *
- * @param msg Header of the message to fire away.
- * @param ch Channel on which the message should go.
- * @param force Is this a forced (undroppable) message?
+ * @param cls the `struct CadetChannel` where one of the ends is now dead
  */
 static void
-fire_and_forget (const struct GNUNET_MessageHeader *msg,
-                 struct CadetChannel *ch,
-                 int force)
+signal_remote_destroy_cb (void *cls)
 {
-  GNUNET_break (NULL ==
-                GCT_send_prebuilt_message (msg, ch->t, NULL,
-                                           force, NULL, NULL));
+  struct CadetChannel *ch = cls;
+  struct CadetChannelClient *ccc;
+
+  /* Find which end is left... */
+  ch->retry_control_task = NULL;
+  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
+  GSC_handle_remote_channel_destroy (ccc->c,
+                                     ccc->ccn,
+                                     ch);
+  channel_destroy (ch);
 }
 
 
 /**
- * Notify that a channel create didn't succeed.
+ * Destroy locally created channel.  Called by the local client, so no
+ * need to tell the client.
  *
- * @param ch The channel to reject.
+ * @param ch channel to destroy
+ * @param c client that caused the destruction
+ * @param ccn client number of the client @a c
  */
-static void
-send_nack (struct CadetChannel *ch)
+void
+GCCH_channel_local_destroy (struct CadetChannel *ch,
+                            struct CadetClient *c,
+                            struct GNUNET_CADET_ClientChannelNumber ccn)
 {
-  struct GNUNET_CADET_ChannelManageMessage msg;
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "  sending channel NACK for channel %s\n",
+       "%s asks for destruction of %s\n",
+       GSC_2s (c),
        GCCH_2s (ch));
+  GNUNET_assert (NULL != c);
+  if ( (NULL != ch->owner) &&
+       (c == ch->owner->c) &&
+       (ccn.channel_of_client == ch->owner->ccn.channel_of_client) )
+  {
+    free_channel_client (ch->owner);
+    ch->owner = NULL;
+  }
+  else if ( (NULL != ch->dest) &&
+            (c == ch->dest->c) &&
+            (ccn.channel_of_client == ch->dest->ccn.channel_of_client) )
+  {
+    free_channel_client (ch->dest);
+    ch->dest = NULL;
+  }
+  else
+  {
+    GNUNET_assert (0);
+  }
 
-  msg.ctn = ch->gid;
-  GCCH_send_prebuilt_message (&msg.header, ch, GNUNET_NO, NULL);
-}
-
-
-/**
- * Destroy all reliable messages queued for a channel,
- * during a channel destruction.
- * Frees the reliability structure itself.
- *
- * @param rel Reliability data for a channel.
- */
-static void
-channel_rel_free_all (struct CadetChannelReliability *rel)
-{
-  struct CadetReliableMessage *copy;
-  struct CadetReliableMessage *next;
-
-  if (NULL == rel)
-    return;
-
-  for (copy = rel->head_recv; NULL != copy; copy = next)
+  if (GNUNET_YES == ch->destroy)
   {
-    next = copy->next;
-    GNUNET_CONTAINER_DLL_remove (rel->head_recv, rel->tail_recv, copy);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " COPYFREE ALL RECV %p\n", copy);
-    GNUNET_break (NULL == copy->chq);
-    GNUNET_free (copy);
+    /* other end already destroyed, with the local client gone, no need
+       to finish transmissions, just destroy immediately. */
+    channel_destroy (ch);
+    return;
   }
-  for (copy = rel->head_sent; NULL != copy; copy = next)
+  if ( (NULL != ch->head_sent) &&
+       ( (NULL != ch->owner) ||
+         (NULL != ch->dest) ) )
   {
-    next = copy->next;
-    GNUNET_CONTAINER_DLL_remove (rel->head_sent, rel->tail_sent, copy);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " COPYFREE ALL SEND %p\n", copy);
-    if (NULL != copy->chq)
-    {
-      if (NULL != copy->chq->tq)
-      {
-        GCT_cancel (copy->chq->tq);
-        /* ch_message_sent will free copy->q */
-      }
-      else
-      {
-        GNUNET_free (copy->chq);
-        GNUNET_break (0);
-      }
-    }
-    GNUNET_free (copy);
+    /* Wait for other end to destroy us as well,
+       and otherwise allow send queue to be transmitted first */
+    ch->destroy = GNUNET_YES;
+    return;
   }
-  if (NULL != rel->uniq && NULL != rel->uniq->tq)
+  if ( (GNUNET_YES == ch->is_loopback) &&
+       ( (NULL != ch->owner) ||
+         (NULL != ch->dest) ) )
   {
-    GCT_cancel (rel->uniq->tq);
-    /* ch_message_sent is called freeing uniq */
+    if (NULL != ch->retry_control_task)
+      GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+    ch->retry_control_task
+      = GNUNET_SCHEDULER_add_now (&signal_remote_destroy_cb,
+                                  ch);
+    return;
   }
-  if (NULL != rel->retry_task)
+  if (GNUNET_NO == ch->is_loopback)
   {
-    GNUNET_SCHEDULER_cancel (rel->retry_task);
-    rel->retry_task = NULL;
+    /* If the we ever sent the CHANNEL_CREATE, we need to send a destroy message. */
+    switch (ch->state)
+    {
+    case CADET_CHANNEL_NEW:
+      /* We gave up on a channel that we created as a client to a remote
+         target, but that never went anywhere. Nothing to do here. */
+      break;
+    case CADET_CHANNEL_LOOSE:
+      GSC_drop_loose_channel (&ch->port,
+                              ch);
+      break;
+    default:
+      GCT_send_channel_destroy (ch->t,
+                                ch->ctn);
+    }
   }
-  GNUNET_free (rel);
+  /* Nothing left to do, just finish destruction */
+  channel_destroy (ch);
 }
 
 
 /**
- * Mark future messages as ACK'd.
- *
- * @param rel Reliability data.
- * @param msg DataACK message with a bitfield of future ACK'd messages.
+ * We got an acknowledgement for the creation of the channel
+ * (the port is open on the other side). Begin transmissions.
  *
- * @return How many messages have been freed.
+ * @param ch channel to destroy
+ * @param cti identifier of the connection that delivered the message
  */
-static unsigned int
-channel_rel_free_sent (struct CadetChannelReliability *rel,
-                       const struct GNUNET_CADET_ChannelDataAckMessage *msg)
+void
+GCCH_handle_channel_open_ack (struct CadetChannel *ch,
+                              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti)
 {
-  struct CadetReliableMessage *copy;
-  struct CadetReliableMessage *next;
-  uint64_t bitfield;
-  uint64_t mask;
-  uint32_t mid;
-  uint32_t target;
-  unsigned int i;
-  unsigned int r;
-
-  bitfield = msg->futures;
-  mid = ntohl (msg->mid);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "free_sent_reliable %u %lX\n", mid, bitfield);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " rel %p, head %p\n", rel, rel->head_sent);
-  for (i = 0, r = 0, copy = rel->head_sent;
-       i < 64 && NULL != copy && 0 != bitfield;
-       i++)
+  switch (ch->state)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " trying bit %u (mid %u)\n", i, mid + i + 1);
-    mask = 0x1LL << i;
-    if (0 == (bitfield & mask))
-     continue;
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " set!\n");
-    /* Bit was set, clear the bit from the bitfield */
-    bitfield &= ~mask;
-
-    /* The i-th bit was set. Do we have that copy? */
-    /* Skip copies with mid < target */
-    target = mid + i + 1;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " target %u\n", target);
-    while (NULL != copy && GC_is_pid_bigger (target, copy->mid))
-      copy = copy->next;
-
-    /* Did we run out of copies? (previously freed, it's ok) */
-    if (NULL == copy)
+  case CADET_CHANNEL_NEW:
+    /* this should be impossible */
+    GNUNET_break (0);
+    break;
+  case CADET_CHANNEL_LOOSE:
+    /* This makes no sense. */
+    GNUNET_break_op (0);
+    break;
+  case CADET_CHANNEL_OPEN_SENT:
+    if (NULL == ch->owner)
     {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "run out of copies...\n");
-      return r;
+      /* We're not the owner, wrong direction! */
+      GNUNET_break_op (0);
+      return;
     }
-
-    /* Did we overshoot the target? (previously freed, it's ok) */
-    if (GC_is_pid_bigger (copy->mid, target))
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received CHANNEL_OPEN_ACK for waiting %s, entering READY state\n",
+         GCCH_2s (ch));
+    if (NULL != ch->retry_control_task) /* can be NULL if ch->is_loopback */
     {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " next copy %u\n", copy->mid);
-      i += copy->mid - target - 1;   /* MID: 90, t = 85, i += 4 (i++ later) */
-      mask = (0x1LL << (i + 1)) - 1; /* Mask = i-th bit and all before */
-      bitfield &= ~mask;             /* Clear all bits up to MID - 1 */
-      continue;
+      GNUNET_SCHEDULER_cancel (ch->retry_control_task);
+      ch->retry_control_task = NULL;
     }
-
-    /* Now copy->mid == target, free it */
-    next = copy->next;
-    GNUNET_break (GNUNET_YES != rel_message_free (copy, GNUNET_YES));
-    r++;
-    copy = next;
+    ch->state = CADET_CHANNEL_READY;
+    /* On first connect, send client as many ACKs as we allow messages
+       to be buffered! */
+    for (unsigned int i=0;i<ch->max_pending_messages;i++)
+      send_ack_to_client (ch,
+                          GNUNET_YES);
+    break;
+  case CADET_CHANNEL_READY:
+    /* duplicate ACK, maybe we retried the CREATE. Ignore. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Received duplicate channel OPEN_ACK for %s\n",
+         GCCH_2s (ch));
+    GNUNET_STATISTICS_update (stats,
+                              "# duplicate CREATE_ACKs",
+                              1,
+                              GNUNET_NO);
+    break;
   }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "free_sent_reliable END\n");
-  return r;
 }
 
 
 /**
- * Destroy a reliable message after it has been acknowledged, either by
- * direct mid ACK or bitfield. Updates the appropriate data structures and
- * timers and frees all memory.
+ * Test if element @a e1 comes before element @a e2.
  *
- * @param copy Message that is no longer needed: remote peer got it.
- * @param update_time Is the timing information relevant?
- *                    If this message is ACK in a batch the timing information
- *                    is skewed by the retransmission, count only for the
- *                    retransmitted message.
- *
- * @return #GNUNET_YES if channel was destroyed as a result of the call,
- *         #GNUNET_NO otherwise.
+ * @param cls closure, to a flag where we indicate duplicate packets
+ * @param m1 a message of to sort
+ * @param m2 another message to sort
+ * @return #GNUNET_YES if @e1 < @e2, otherwise #GNUNET_NO
  */
 static int
-rel_message_free (struct CadetReliableMessage *copy, int update_time)
+is_before (void *cls,
+           struct CadetOutOfOrderMessage *m1,
+           struct CadetOutOfOrderMessage *m2)
 {
-  struct CadetChannelReliability *rel;
-  struct GNUNET_TIME_Relative time;
+  int *duplicate = cls;
+  uint32_t v1 = ntohl (m1->mid.mid);
+  uint32_t v2 = ntohl (m2->mid.mid);
+  uint32_t delta;
 
-  rel = copy->rel;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Freeing %u\n", copy->mid);
-  if (GNUNET_YES == update_time)
+  delta = v2 - v1;
+  if (0 == delta)
+    *duplicate = GNUNET_YES;
+  if (delta > (uint32_t) INT_MAX)
   {
-    time = GNUNET_TIME_absolute_get_duration (copy->timestamp);
-    if (0 == rel->expected_delay.rel_value_us)
-      rel->expected_delay = time;
-    else
-    {
-      rel->expected_delay.rel_value_us *= 7;
-      rel->expected_delay.rel_value_us += time.rel_value_us;
-      rel->expected_delay.rel_value_us /= 8;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  message time %12s\n",
-         GNUNET_STRINGS_relative_time_to_string (time, GNUNET_NO));
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  new delay    %12s\n",
-         GNUNET_STRINGS_relative_time_to_string (rel->expected_delay,
-                                                 GNUNET_NO));
-    rel->retry_timer = rel->expected_delay;
+    /* in overflow range, we can safely assume we wrapped around */
+    return GNUNET_NO;
   }
   else
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "batch free, ignoring timing\n");
-  }
-  rel->ch->pending_messages--;
-  if (NULL != copy->chq)
-  {
-    GCT_cancel (copy->chq->tq);
-    /* copy->q is set to NULL by ch_message_sent */
-  }
-  GNUNET_CONTAINER_DLL_remove (rel->head_sent, rel->tail_sent, copy);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " free send copy MID %u at %p\n",
-       copy->mid, copy);
-  GNUNET_free (copy);
-
-  if (GNUNET_NO != rel->ch->destroy && 0 == rel->ch->pending_messages)
-  {
-    GCCH_destroy (rel->ch);
+    /* result is small, thus v2 > v1, thus m1 < m2 */
     return GNUNET_YES;
   }
-  return GNUNET_NO;
 }
 
 
 /**
- * Channel was ACK'd by remote peer, mark as ready and cancel retransmission.
+ * We got payload data for a channel.  Pass it on to the client
+ * and send an ACK to the other end (once flow control allows it!)
  *
- * @param ch Channel to mark as ready.
- * @param fwd Was the ACK message a FWD ACK? (dest->root, SYNACK)
+ * @param ch channel that got data
+ * @param cti identifier of the connection that delivered the message
+ * @param msg message that was received
  */
-static void
-channel_confirm (struct CadetChannel *ch, int fwd)
+void
+GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
+                                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                                    const struct GNUNET_CADET_ChannelAppDataMessage *msg)
 {
-  struct CadetChannelReliability *rel;
-  enum CadetChannelState oldstate;
-
-  rel = fwd ? ch->root_rel : ch->dest_rel;
-  if (NULL == rel)
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_LocalData *ld;
+  struct CadetChannelClient *ccc;
+  size_t payload_size;
+  struct CadetOutOfOrderMessage *com;
+  int duplicate;
+  uint32_t mid_min;
+  uint32_t mid_max;
+  uint32_t mid_msg;
+  uint32_t delta;
+
+  GNUNET_assert (GNUNET_NO == ch->is_loopback);
+  if ( (GNUNET_YES == ch->destroy) &&
+       (NULL == ch->owner) &&
+       (NULL == ch->dest) )
+  {
+    /* This client is gone, but we still have messages to send to
+       the other end (which is why @a ch is not yet dead).  However,
+       we cannot pass messages to our client anymore. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Dropping incoming payload on %s as this end is already closed\n",
+         GCCH_2s (ch));
+    /* send back DESTROY notification to stop further retransmissions! */
+    GCT_send_channel_destroy (ch->t,
+                              ch->ctn);
+    return;
+  }
+  payload_size = ntohs (msg->header.size) - sizeof (*msg);
+  env = GNUNET_MQ_msg_extra (ld,
+                             payload_size,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
+  ld->ccn = (NULL == ch->dest) ? ch->owner->ccn : ch->dest->ccn;
+  GNUNET_memcpy (&ld[1],
+                 &msg[1],
+                 payload_size);
+  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
+  if ( (GNUNET_YES == ccc->client_ready) &&
+       ( (GNUNET_YES == ch->out_of_order) ||
+         (msg->mid.mid == ch->mid_recv.mid) ) )
   {
-    GNUNET_break (GNUNET_NO != ch->destroy);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Giving %u bytes of payload with MID %u from %s to client %s\n",
+         (unsigned int) payload_size,
+         ntohl (msg->mid.mid),
+         GCCH_2s (ch),
+         GSC_2s (ccc->c));
+    ccc->client_ready = GNUNET_NO;
+    GSC_send_to_client (ccc->c,
+                        env);
+    ch->mid_recv.mid = htonl (1 + ntohl (ch->mid_recv.mid));
+    ch->mid_futures >>= 1;
+    send_channel_data_ack (ch);
     return;
   }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  channel confirm %s %s\n",
-       GC_f2s (fwd), GCCH_2s (ch));
-  oldstate = ch->state;
-  ch->state = CADET_CHANNEL_READY;
 
-  if (CADET_CHANNEL_READY != oldstate || GNUNET_YES == is_loopback (ch))
+  if (GNUNET_YES == ch->reliable)
   {
-    rel->client_ready = GNUNET_YES;
-    rel->expected_delay = rel->retry_timer;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  confirm retry timer %s\n",
-         GNUNET_STRINGS_relative_time_to_string (rel->retry_timer, GNUNET_NO));
-    if (GCT_get_connections_buffer (ch->t) > 0 || GCT_is_loopback (ch->t))
-      send_client_ack (ch, fwd);
-
-    if (NULL != rel->retry_task)
+    /* check if message ought to be dropped because it is ancient/too distant/duplicate */
+    mid_min = ntohl (ch->mid_recv.mid);
+    mid_max = mid_min + ch->max_pending_messages;
+    mid_msg = ntohl (msg->mid.mid);
+    if ( ( (uint32_t) (mid_msg - mid_min) > ch->max_pending_messages) ||
+         ( (uint32_t) (mid_max - mid_msg) > ch->max_pending_messages) )
     {
-      GNUNET_SCHEDULER_cancel (rel->retry_task);
-      rel->retry_task = NULL;
-    }
-    else if (NULL != rel->uniq)
-    {
-      GCT_cancel (rel->uniq->tq);
-      /* ch_message_sent will free and NULL uniq */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "%s at %u drops ancient or far-future message %u\n",
+           GCCH_2s (ch),
+           (unsigned int) mid_min,
+           ntohl (msg->mid.mid));
+
+      GNUNET_STATISTICS_update (stats,
+                                "# duplicate DATA (ancient or future)",
+                                1,
+                                GNUNET_NO);
+      GNUNET_MQ_discard (env);
+      send_channel_data_ack (ch);
+      return;
     }
-    else if (GNUNET_NO == is_loopback (ch))
+    /* mark bit for future ACKs */
+    delta = mid_msg - mid_min - 1; /* overflow/underflow are OK here */
+    if (delta < 64)
     {
-      /* We SHOULD have been trying to retransmit this! */
-      GNUNET_break (0);
+      if (0 != (ch->mid_futures & (1LLU << delta)))
+      {
+        /* Duplicate within the queue, drop also */
+        LOG (GNUNET_ERROR_TYPE_DEBUG,
+             "Duplicate payload of %u bytes on %s (mid %u) dropped\n",
+             (unsigned int) payload_size,
+             GCCH_2s (ch),
+             ntohl (msg->mid.mid));
+        GNUNET_STATISTICS_update (stats,
+                                  "# duplicate DATA",
+                                  1,
+                                  GNUNET_NO);
+        GNUNET_MQ_discard (env);
+        send_channel_data_ack (ch);
+        return;
+      }
+      ch->mid_futures |= (1LLU << delta);
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Marked bit %llX for mid %u (base: %u); now: %llX\n",
+           (1LLU << delta),
+           mid_msg,
+           mid_min,
+           ch->mid_futures);
     }
   }
-
-  /* In case of a FWD ACK (SYNACK) send a BCK ACK (ACK). */
-  if (GNUNET_YES == fwd)
-    send_ack (ch, GNUNET_NO);
-}
-
-
-/**
- * Save a copy to retransmit in case it gets lost.
- *
- * Initializes all needed callbacks and timers.
- *
- * @param ch Channel this message goes on.
- * @param msg Message to copy.
- * @param fwd Is this fwd traffic?
- */
-static struct CadetReliableMessage *
-channel_save_copy (struct CadetChannel *ch,
-                   const struct GNUNET_MessageHeader *msg,
-                   int fwd)
-{
-  struct CadetChannelReliability *rel;
-  struct CadetReliableMessage *copy;
-  uint32_t mid;
-  uint16_t type;
-  uint16_t size;
-
-  rel = fwd ? ch->root_rel : ch->dest_rel;
-  mid = rel->mid_send - 1;
-  type = ntohs (msg->type);
-  size = ntohs (msg->size);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "save MID %u %s\n", mid, GC_m2s (type));
-  copy = GNUNET_malloc (sizeof (struct CadetReliableMessage) + size);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  at %p\n", copy);
-  copy->mid = mid;
-  copy->rel = rel;
-  copy->type = type;
-  GNUNET_memcpy (&copy[1], msg, size);
-  GNUNET_CONTAINER_DLL_insert_tail (rel->head_sent, rel->tail_sent, copy);
-  ch->pending_messages++;
-
-  return copy;
-}
-
-
-/**
- * Create a new channel.
- *
- * @param t Tunnel this channel is in.
- * @param owner Client that owns the channel, NULL for foreign channels.
- * @param lid_root Local ID for root client.
- *
- * @return A new initialized channel. NULL on error.
- */
-static struct CadetChannel *
-channel_new (struct CadetTunnel *t,
-             struct CadetClient *owner,
-             struct GNUNET_CADET_ClientChannelNumber lid_root)
-{
-  struct CadetChannel *ch;
-
-  ch = GNUNET_new (struct CadetChannel);
-  ch->root = owner;
-  ch->lid_root = lid_root;
-  ch->t = t;
-
-  GNUNET_STATISTICS_update (stats, "# channels", 1, GNUNET_NO);
-
-  if (NULL != owner)
-  {
-    ch->gid = GCT_get_next_ctn (t);
-    GML_channel_add (owner, lid_root, ch);
-  }
-  GCT_add_channel (t, ch);
-
-  return ch;
-}
-
-
-/**
- * Handle a loopback message: call the appropriate handler for the message type.
- *
- * @param ch Channel this message is on.
- * @param msgh Message header.
- * @param fwd Is this FWD traffic?
- */
-void
-handle_loopback (struct CadetChannel *ch,
-                 const struct GNUNET_MessageHeader *msgh,
-                 int fwd)
-{
-  uint16_t type;
-
-  type = ntohs (msgh->type);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Loopback %s %s message!\n",
-       GC_f2s (fwd), GC_m2s (type));
-
-  switch (type)
+  else /* ! ch->reliable */
   {
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-      /* Don't send hop ACK, wait for client to ACK */
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "SEND loopback %u (%u)\n",
-           ntohl (((struct GNUNET_CADET_ChannelAppDataMessage *) msgh)->mid), ntohs (msgh->size));
-      GCCH_handle_data (ch, (struct GNUNET_CADET_ChannelAppDataMessage *) msgh, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-      GCCH_handle_data_ack (ch,
-                            (const struct GNUNET_CADET_ChannelDataAckMessage *) msgh, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-      GCCH_handle_create (ch->t,
-                          (const struct GNUNET_CADET_ChannelOpenMessage *) msgh);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-      GCCH_handle_ack (ch,
-                       (const struct GNUNET_CADET_ChannelManageMessage *) msgh,
-                       fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED:
-      GCCH_handle_nack (ch);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-      GCCH_handle_destroy (ch,
-                           (const struct GNUNET_CADET_ChannelManageMessage *) msgh,
-                           fwd);
-      break;
+    /* Channel is unreliable, so we do not ACK. But we also cannot
+       allow buffering everything, so check if we have space... */
+    if (ccc->num_recv >= ch->max_pending_messages)
+    {
+      struct CadetOutOfOrderMessage *drop;
 
-    default:
-      GNUNET_break_op (0);
+      /* Yep, need to drop. Drop the oldest message in
+         the buffer. */
       LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "end-to-end message not known (%u)\n",
-           ntohs (msgh->type));
+           "Queue full due slow client on %s, dropping oldest message\n",
+           GCCH_2s (ch));
+      GNUNET_STATISTICS_update (stats,
+                                "# messages dropped due to slow client",
+                                1,
+                                GNUNET_NO);
+      drop = ccc->head_recv;
+      GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                                   ccc->tail_recv,
+                                   drop);
+      ccc->num_recv--;
+      GNUNET_MQ_discard (drop->env);
+      GNUNET_free (drop);
+    }
   }
-}
-
-
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Destroy a channel and free all resources.
- *
- * @param ch Channel to destroy.
- */
-void
-GCCH_destroy (struct CadetChannel *ch)
-{
-  struct CadetClient *c;
-  struct CadetTunnel *t;
 
-  if (NULL == ch)
+  /* Insert message into sorted out-of-order queue */
+  com = GNUNET_new (struct CadetOutOfOrderMessage);
+  com->mid = msg->mid;
+  com->env = env;
+  duplicate = GNUNET_NO;
+  GNUNET_CONTAINER_DLL_insert_sorted (struct CadetOutOfOrderMessage,
+                                      is_before,
+                                      &duplicate,
+                                      ccc->head_recv,
+                                      ccc->tail_recv,
+                                      com);
+  ccc->num_recv++;
+  if (GNUNET_YES == duplicate)
+  {
+    /* Duplicate within the queue, drop also (this is not covered by
+       the case above if "delta" >= 64, which could be the case if
+       max_pending_messages is also >= 64 or if our client is unready
+       and we are seeing retransmissions of the message our client is
+       blocked on. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Duplicate payload of %u bytes on %s (mid %u) dropped\n",
+         (unsigned int) payload_size,
+         GCCH_2s (ch),
+         ntohl (msg->mid.mid));
+    GNUNET_STATISTICS_update (stats,
+                              "# duplicate DATA",
+                              1,
+                              GNUNET_NO);
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                                 ccc->tail_recv,
+                                 com);
+    ccc->num_recv--;
+    GNUNET_MQ_discard (com->env);
+    GNUNET_free (com);
+    send_channel_data_ack (ch);
     return;
-  if (2 == ch->destroy)
-    return; /* recursive call */
-  ch->destroy = 2;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "destroying channel %s:%u\n",
-              GCT_2s (ch->t), ch->gid);
-  GCCH_debug (ch, GNUNET_ERROR_TYPE_DEBUG);
-
-  c = ch->root;
-  if (NULL != c)
-  {
-    GML_channel_remove (c, ch->lid_root, ch);
-  }
-
-  c = ch->dest;
-  if (NULL != c)
-  {
-    GML_channel_remove (c, ch->lid_dest, ch);
   }
-
-  channel_rel_free_all (ch->root_rel);
-  channel_rel_free_all (ch->dest_rel);
-
-  t = ch->t;
-  GCT_remove_channel (t, ch);
-  GNUNET_STATISTICS_update (stats, "# channels", -1, GNUNET_NO);
-
-  GNUNET_free (ch);
-  GCT_destroy_if_empty (t);
-}
-
-
-/**
- * Get the channel's public ID.
- *
- * @param ch Channel.
- *
- * @return ID used to identify the channel with the remote peer.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCCH_get_id (const struct CadetChannel *ch)
-{
-  return ch->gid;
-}
-
-
-/**
- * Get the channel tunnel.
- *
- * @param ch Channel to get the tunnel from.
- *
- * @return tunnel of the channel.
- */
-struct CadetTunnel *
-GCCH_get_tunnel (const struct CadetChannel *ch)
-{
-  return ch->t;
-}
-
-
-/**
- * Get free buffer space towards the client on a specific channel.
- *
- * @param ch Channel.
- * @param fwd Is query about FWD traffic?
- *
- * @return Free buffer space [0 - 64]
- */
-unsigned int
-GCCH_get_buffer (struct CadetChannel *ch, int fwd)
-{
-  struct CadetChannelReliability *rel;
-
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "   get buffer, channel %s\n", GCCH_2s (ch));
-  GCCH_debug (ch, GNUNET_ERROR_TYPE_DEBUG);
-  /* If rel is NULL it means that the end is not yet created,
-   * most probably is a loopback channel at the point of sending
-   * the ChannelCreate to itself.
-   */
-  if (NULL == rel)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  rel is NULL: max\n");
-    return 64;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "   n_recv %d\n", rel->n_recv);
-  return (64 - rel->n_recv);
-}
-
-
-/**
- * Get flow control status of end point: is client allow to send?
- *
- * @param ch Channel.
- * @param fwd Is query about FWD traffic? (Request root status).
- *
- * @return #GNUNET_YES if client is allowed to send us data.
- */
-int
-GCCH_get_allowed (struct CadetChannel *ch, int fwd)
-{
-  struct CadetChannelReliability *rel;
-
-  rel = fwd ? ch->root_rel : ch->dest_rel;
-
-  if (NULL == rel)
-  {
-    /* Probably shutting down: root/dest NULL'ed to mark disconnection */
-    GNUNET_break (GNUNET_NO != ch->destroy);
-    return 0;
-  }
-
-  return rel->client_allowed;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Queued %s payload of %u bytes on %s-%X(%p) (mid %u, need %u first)\n",
+       (GNUNET_YES == ccc->client_ready)
+       ? "out-of-order"
+       : "client-not-ready",
+       (unsigned int) payload_size,
+       GCCH_2s (ch),
+       ntohl (ccc->ccn.channel_of_client),
+       ccc,
+       ntohl (msg->mid.mid),
+       ntohl (ch->mid_recv.mid));
+  /* NOTE: this ACK we _could_ skip, as the packet is out-of-order and
+     the sender may already be transmitting the previous one.  Needs
+     experimental evaluation to see if/when this ACK helps or
+     hurts. (We might even want another option.) */
+  send_channel_data_ack (ch);
 }
 
 
 /**
- * Is the root client for this channel on this peer?
- *
- * @param ch Channel.
- * @param fwd Is this for fwd traffic?
- *
- * @return #GNUNET_YES in case it is.
+ * Function called once the tunnel has sent one of our messages.
+ * If the message is unreliable, simply frees the `crm`. If the
+ * message was reliable, calculate retransmission time and
+ * wait for ACK (or retransmit).
+ *
+ * @param cls the `struct CadetReliableMessage` that was sent
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
  */
-int
-GCCH_is_origin (struct CadetChannel *ch, int fwd)
-{
-  struct CadetClient *c;
-
-  c = fwd ? ch->root : ch->dest;
-  return NULL != c;
-}
+static void
+data_sent_cb (void *cls,
+              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
 
 
 /**
- * Is the destination client for this channel on this peer?
- *
- * @param ch Channel.
- * @param fwd Is this for fwd traffic?
+ * We need to retry a transmission, the last one took too long to
+ * be acknowledged.
  *
- * @return #GNUNET_YES in case it is.
+ * @param cls the `struct CadetChannel` where we need to retransmit
  */
-int
-GCCH_is_terminal (struct CadetChannel *ch, int fwd)
+static void
+retry_transmission (void *cls)
 {
-  struct CadetClient *c;
+  struct CadetChannel *ch = cls;
+  struct CadetReliableMessage *crm = ch->head_sent;
 
-  c = fwd ? ch->dest : ch->root;
-  return NULL != c;
+  ch->retry_data_task = NULL;
+  GNUNET_assert (NULL == crm->qe);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Retrying transmission on %s of message %u\n",
+       GCCH_2s (ch),
+       (unsigned int) ntohl (crm->data_message->mid.mid));
+  crm->qe = GCT_send (ch->t,
+                      &crm->data_message->header,
+                      &data_sent_cb,
+                      crm);
+  GNUNET_assert (NULL == ch->retry_data_task);
 }
 
 
 /**
- * Send an end-to-end ACK message for the most recent in-sequence payload.
+ * We got an PLAINTEXT_DATA_ACK for a message in our queue, remove it from
+ * the queue and tell our client that it can send more.
  *
- * If channel is not reliable, do nothing.
- *
- * @param ch Channel this is about.
- * @param fwd Is for FWD traffic? (ACK dest->owner)
+ * @param ch the channel that got the PLAINTEXT_DATA_ACK
+ * @param cti identifier of the connection that delivered the message
+ * @param crm the message that got acknowledged
  */
-void
-GCCH_send_data_ack (struct CadetChannel *ch, int fwd)
+static void
+handle_matching_ack (struct CadetChannel *ch,
+                     const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                     struct CadetReliableMessage *crm)
 {
-  struct GNUNET_CADET_ChannelDataAckMessage msg;
-  struct CadetChannelReliability *rel;
-  struct CadetReliableMessage *copy;
-  unsigned int delta;
-  uint64_t mask;
-  uint32_t ack;
-
-  if (GNUNET_NO == ch->reliable)
-    return;
-
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-  ack = rel->mid_recv - 1;
-
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK);
-  msg.header.size = htons (sizeof (msg));
-  msg.ctn = ch->gid;
-  msg.mid = htonl (ack);
-
-  msg.futures = 0LL;
-  for (copy = rel->head_recv; NULL != copy; copy = copy->next)
-  {
-    if (copy->type != GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA)
+  GNUNET_CONTAINER_DLL_remove (ch->head_sent,
+                               ch->tail_sent,
+                               crm);
+  ch->pending_messages--;
+  GNUNET_assert (ch->pending_messages < ch->max_pending_messages);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received DATA_ACK on %s for message %u (%u ACKs pending)\n",
+       GCCH_2s (ch),
+       (unsigned int) ntohl (crm->data_message->mid.mid),
+       ch->pending_messages);
+  if (NULL != crm->qe)
+  {
+    GCT_send_cancel (crm->qe);
+    crm->qe = NULL;
+  }
+  if ( (1 == crm->num_transmissions) &&
+       (NULL != cti) )
+  {
+    GCC_ack_observed (cti);
+    if (0 == memcmp (cti,
+                     &crm->connection_taken,
+                     sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier)))
     {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " Type %s, expected DATA\n",
-           GC_m2s (copy->type));
-      continue;
+      GCC_latency_observed (cti,
+                            GNUNET_TIME_absolute_get_duration (crm->first_transmission_time));
     }
-    GNUNET_assert (GC_is_pid_bigger(copy->mid, ack));
-    delta = copy->mid - (ack + 1);
-    if (63 < delta)
-      break;
-    mask = 0x1LL << delta;
-    msg.futures |= mask;
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         " setting bit for %u (delta %u) (%lX) -> %lX\n",
-         copy->mid, delta, mask, msg.futures);
   }
-
-  GCCH_send_prebuilt_message (&msg.header, ch, !fwd, NULL);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "send_data_ack END\n");
+  GNUNET_free (crm->data_message);
+  GNUNET_free (crm);
+  send_ack_to_client (ch,
+                      (NULL == ch->owner)
+                      ? GNUNET_NO
+                      : GNUNET_YES);
 }
 
 
 /**
- * Allow a client to send us more data, in case it was choked.
+ * We got an acknowledgement for payload data for a channel.
+ * Possibly resume transmissions.
  *
- * @param ch Channel.
- * @param fwd Is this about FWD traffic? (Root client).
+ * @param ch channel that got the ack
+ * @param cti identifier of the connection that delivered the message
+ * @param ack details about what was received
  */
 void
-GCCH_allow_client (struct CadetChannel *ch, int fwd)
+GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
+                                        const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                                        const struct GNUNET_CADET_ChannelDataAckMessage *ack)
 {
-  struct CadetChannelReliability *rel;
-  unsigned int buffer;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GMCH allow\n");
+  struct CadetReliableMessage *crm;
+  struct CadetReliableMessage *crmn;
+  int found;
+  uint32_t mid_base;
+  uint64_t mid_mask;
+  unsigned int delta;
 
-  if (CADET_CHANNEL_READY != ch->state)
+  GNUNET_break (GNUNET_NO == ch->is_loopback);
+  if (GNUNET_NO == ch->reliable)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " channel not ready yet!\n");
+    /* not expecting ACKs on unreliable channel, odd */
+    GNUNET_break_op (0);
     return;
   }
-
-  if (GNUNET_YES == ch->reliable)
-  {
-    rel = fwd ? ch->root_rel : ch->dest_rel;
-    if (NULL == rel)
-    {
-      GNUNET_break (GNUNET_NO != ch->destroy);
-      return;
-    }
-    if (NULL != rel->head_sent)
+  /* mid_base is the MID of the next message that the
+     other peer expects (i.e. that is missing!), everything
+     LOWER (but excluding mid_base itself) was received. */
+  mid_base = ntohl (ack->mid.mid);
+  mid_mask = GNUNET_htonll (ack->futures);
+  found = GNUNET_NO;
+  for (crm = ch->head_sent;
+        NULL != crm;
+       crm = crmn)
+  {
+    crmn = crm->next;
+    delta = (unsigned int) (ntohl (crm->data_message->mid.mid) - mid_base);
+    if (delta >= UINT_MAX - ch->max_pending_messages)
     {
-      if (64 <= rel->mid_send - rel->head_sent->mid)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, " too big MID gap! Wait for ACK.\n");
-        return;
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, " gap ok: %u - %u\n",
-             rel->head_sent->mid, rel->mid_send);
-        struct CadetReliableMessage *aux;
-        for (aux = rel->head_sent; NULL != aux; aux = aux->next)
-        {
-          LOG (GNUNET_ERROR_TYPE_DEBUG, "   - sent mid %u\n", aux->mid);
-        }
-      }
+      /* overflow, means crm was a bit in the past, so this ACK counts for it. */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got DATA_ACK with base %u satisfying past message %u on %s\n",
+           (unsigned int) mid_base,
+           ntohl (crm->data_message->mid.mid),
+           GCCH_2s (ch));
+      handle_matching_ack (ch,
+                           cti,
+                           crm);
+      found = GNUNET_YES;
+      continue;
     }
-    else
+    delta--;
+    if (delta >= 64)
+      continue;
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Testing bit %llX for mid %u (base: %u)\n",
+         (1LLU << delta),
+         ntohl (crm->data_message->mid.mid),
+         mid_base);
+    if (0 != (mid_mask & (1LLU << delta)))
     {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " head sent is NULL\n");
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got DATA_ACK with mask for %u on %s\n",
+           ntohl (crm->data_message->mid.mid),
+           GCCH_2s (ch));
+      handle_matching_ack (ch,
+                           cti,
+                           crm);
+      found = GNUNET_YES;
     }
   }
-
-  if (is_loopback (ch))
-    buffer = GCCH_get_buffer (ch, fwd);
-  else
-    buffer = GCT_get_connections_buffer (ch->t);
-
-  if (0 == buffer)
+  if (GNUNET_NO == found)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " no buffer space.\n");
+    /* ACK for message we already dropped, might have been a
+       duplicate ACK? Ignore. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Duplicate DATA_ACK on %s, ignoring\n",
+         GCCH_2s (ch));
+    GNUNET_STATISTICS_update (stats,
+                              "# duplicate DATA_ACKs",
+                              1,
+                              GNUNET_NO);
     return;
   }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " buffer space %u, allowing\n", buffer);
-  send_client_ack (ch, fwd);
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
+  if ( (NULL != ch->head_sent) &&
+       (NULL == ch->head_sent->qe) )
+    ch->retry_data_task
+      = GNUNET_SCHEDULER_add_at (ch->head_sent->next_retry,
+                                 &retry_transmission,
+                                 ch);
 }
 
 
 /**
- * Log channel info.
+ * Destroy channel, based on the other peer closing the
+ * connection.  Also needs to remove this channel from
+ * the tunnel.
  *
- * @param ch Channel.
- * @param level Debug level to use.
+ * @param ch channel to destroy
+ * @param cti identifier of the connection that delivered the message,
+ *            NULL if we are simulating receiving a destroy due to shutdown
  */
 void
-GCCH_debug (struct CadetChannel *ch, enum GNUNET_ErrorType level)
+GCCH_handle_remote_destroy (struct CadetChannel *ch,
+                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti)
 {
-  int do_log;
+  struct CadetChannelClient *ccc;
 
-  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
-                                       "cadet-chn",
-                                       __FILE__, __FUNCTION__, __LINE__);
-  if (0 == do_log)
-    return;
-
-  if (NULL == ch)
+  GNUNET_assert (GNUNET_NO == ch->is_loopback);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received remote channel DESTROY for %s\n",
+       GCCH_2s (ch));
+  if (GNUNET_YES == ch->destroy)
   {
-    LOG2 (level, "CHN *** DEBUG NULL CHANNEL ***\n");
+    /* Local client already gone, this is instant-death. */
+    channel_destroy (ch);
     return;
   }
-  LOG2 (level, "CHN Channel %s:%X (%p)\n", GCT_2s (ch->t), ch->gid, ch);
-  LOG2 (level, "CHN   root %p/%p\n", ch->root, ch->root_rel);
-  if (NULL != ch->root)
-  {
-    LOG2 (level, "CHN   cli %s\n", GML_2s (ch->root));
-    LOG2 (level, "CHN   ready %s\n", ch->root_rel->client_ready ? "YES" : "NO");
-    LOG2 (level, "CHN   id %X\n", ch->lid_root.channel_of_client);
-    LOG2 (level, "CHN   recv %d\n", ch->root_rel->n_recv);
-    LOG2 (level, "CHN   MID r: %d, s: %d\n",
-          ch->root_rel->mid_recv, ch->root_rel->mid_send);
-  }
-  LOG2 (level, "CHN   dest %p/%p\n",
-              ch->dest, ch->dest_rel);
-  if (NULL != ch->dest)
+  ccc = (NULL != ch->owner) ? ch->owner : ch->dest;
+  if ( (NULL != ccc) &&
+       (NULL != ccc->head_recv) )
   {
-    LOG2 (level, "CHN   cli %s\n", GML_2s (ch->dest));
-    LOG2 (level, "CHN   ready %s\n", ch->dest_rel->client_ready ? "YES" : "NO");
-    LOG2 (level, "CHN   id %X\n", ch->lid_dest);
-    LOG2 (level, "CHN   recv %d\n", ch->dest_rel->n_recv);
-    LOG2 (level, "CHN   MID r: %d, s: %d\n",
-          ch->dest_rel->mid_recv, ch->dest_rel->mid_send);
-
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Lost end of transmission due to remote shutdown on %s\n",
+         GCCH_2s (ch));
+    /* FIXME: change API to notify client about truncated transmission! */
   }
+  ch->destroy = GNUNET_YES;
+  if (NULL != ccc)
+    GSC_handle_remote_channel_destroy (ccc->c,
+                                       ccc->ccn,
+                                       ch);
+  channel_destroy (ch);
 }
 
 
 /**
- * Handle an ACK given by a client.
- *
- * Mark client as ready and send him any buffered data we could have for him.
+ * Test if element @a e1 comes before element @a e2.
  *
- * @param ch Channel.
- * @param fwd Is this a "FWD ACK"? (FWD ACKs are sent by dest and go BCK)
+ * @param cls closure, to a flag where we indicate duplicate packets
+ * @param crm1 an element of to sort
+ * @param crm2 another element to sort
+ * @return #GNUNET_YES if @e1 < @e2, otherwise #GNUNET_NO
  */
-void
-GCCH_handle_local_ack (struct CadetChannel *ch, int fwd)
+static int
+cmp_crm_by_next_retry (void *cls,
+                       struct CadetReliableMessage *crm1,
+                       struct CadetReliableMessage *crm2)
 {
-  struct CadetChannelReliability *rel;
-  struct CadetClient *c;
-
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-  c   = fwd ? ch->dest     : ch->root;
-
-  rel->client_ready = GNUNET_YES;
-  send_client_buffered_data (ch, c, fwd);
-
-  if (GNUNET_YES == ch->destroy && 0 == rel->n_recv)
-  {
-    send_destroy (ch, GNUNET_YES);
-    GCCH_destroy (ch);
-    return;
-  }
-  /* if loopback is marked for destruction, no need to ACK to the other peer,
-   * it requested the destruction and is already gone, therefore, else if.
-   */
-  else if (is_loopback (ch))
-  {
-    unsigned int buffer;
-
-    buffer = GCCH_get_buffer (ch, fwd);
-    if (0 < buffer)
-      GCCH_allow_client (ch, fwd);
-
-    return;
-  }
-  GCT_send_connection_acks (ch->t);
+  if (crm1->next_retry.abs_value_us <
+      crm2->next_retry.abs_value_us)
+    return GNUNET_YES;
+  return GNUNET_NO;
 }
 
 
 /**
- * Handle data given by a client.
- *
- * Check whether the client is allowed to send in this tunnel, save if channel
- * is reliable and send an ACK to the client if there is still buffer space
- * in the tunnel.
- *
- * @param ch Channel.
- * @param c Client which sent the data.
- * @param fwd Is this a FWD data?
- * @param message Data message.
- * @param size Size of data.
- *
- * @return #GNUNET_OK if everything goes well, #GNUNET_SYSERR in case of en error.
+ * Function called once the tunnel has sent one of our messages.
+ * If the message is unreliable, simply frees the `crm`. If the
+ * message was reliable, calculate retransmission time and
+ * wait for ACK (or retransmit).
+ *
+ * @param cls the `struct CadetReliableMessage` that was sent
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
  */
-int
-GCCH_handle_local_data (struct CadetChannel *ch,
-                        struct CadetClient *c,
-                        int fwd,
-                        const struct GNUNET_MessageHeader *message,
-                        size_t size)
+static void
+data_sent_cb (void *cls,
+              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  struct CadetChannelReliability *rel;
-  struct GNUNET_CADET_ChannelAppDataMessage *payload;
-  uint16_t p2p_size = sizeof(struct GNUNET_CADET_ChannelAppDataMessage) + size;
-  unsigned char cbuf[p2p_size];
-  unsigned char buffer;
-
-  /* Is the client in the channel? */
-  if ( !( (fwd &&
-           ch->root == c)
-         ||
-          (!fwd &&
-           ch->dest == c) ) )
+  struct CadetReliableMessage *crm = cls;
+  struct CadetChannel *ch = crm->ch;
+
+  GNUNET_assert (GNUNET_NO == ch->is_loopback);
+  GNUNET_assert (NULL != crm->qe);
+  crm->qe = NULL;
+  GNUNET_CONTAINER_DLL_remove (ch->head_sent,
+                               ch->tail_sent,
+                               crm);
+  if (GNUNET_NO == ch->reliable)
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    GNUNET_free (crm->data_message);
+    GNUNET_free (crm);
+    ch->pending_messages--;
+    send_ack_to_client (ch,
+                        (NULL == ch->owner)
+                        ? GNUNET_NO
+                        : GNUNET_YES);
+    return;
   }
-
-  rel = fwd ? ch->root_rel : ch->dest_rel;
-
-  if (GNUNET_NO == rel->client_allowed)
+  if (NULL == cid)
   {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
+    /* There was an error sending. */
+    crm->num_transmissions = GNUNET_SYSERR;
   }
-
-  rel->client_allowed = GNUNET_NO;
-
-  /* Ok, everything is correct, send the message. */
-  payload = (struct GNUNET_CADET_ChannelAppDataMessage *) cbuf;
-  payload->mid = htonl (rel->mid_send);
-  rel->mid_send++;
-  GNUNET_memcpy (&payload[1], message, size);
-  payload->header.size = htons (p2p_size);
-  payload->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA);
-  payload->ctn = ch->gid;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  sending on channel...\n");
-  GCCH_send_prebuilt_message (&payload->header, ch, fwd, NULL);
-
-  if (is_loopback (ch))
-    buffer = GCCH_get_buffer (ch, fwd);
-  else
-    buffer = GCT_get_connections_buffer (ch->t);
-
-  if (0 < buffer)
-    GCCH_allow_client (ch, fwd);
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Handle a channel destroy requested by a client.
- *
- * TODO: add "reason" field
- *
- * Destroy the channel and the tunnel in case this was the last channel.
- *
- * @param ch Channel.
- * @param c Client that requested the destruction (to avoid notifying him).
- * @param is_root Is the request coming from root?
- */
-void
-GCCH_handle_local_destroy (struct CadetChannel *ch,
-                           struct CadetClient *c,
-                           int is_root)
-{
-  ch->destroy = GNUNET_YES;
-  /* Cleanup after the tunnel */
-  if (GNUNET_NO == is_root && c == ch->dest)
+  else if (GNUNET_SYSERR != crm->num_transmissions)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " Client %s is destination.\n", GML_2s (c));
-    GML_client_delete_channel (c, ch, ch->lid_dest);
-    ch->dest = NULL;
+    /* Increment transmission counter, and possibly store @a cid
+       if this was the first transmission. */
+    crm->num_transmissions++;
+    if (1 == crm->num_transmissions)
+    {
+      crm->first_transmission_time = GNUNET_TIME_absolute_get ();
+      crm->connection_taken = *cid;
+      GCC_ack_expected (cid);
+    }
   }
-  if (GNUNET_YES == is_root && c == ch->root)
+  if ( (0 == crm->retry_delay.rel_value_us) &&
+       (NULL != cid) )
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " Client %s is owner.\n", GML_2s (c));
-    GML_client_delete_channel (c, ch, ch->lid_root);
-    ch->root = NULL;
-  }
+    struct CadetConnection *cc = GCC_lookup (cid);
 
-  send_destroy (ch, GNUNET_NO);
-  if (0 == ch->pending_messages)
-    GCCH_destroy (ch);
+    if (NULL != cc)
+      crm->retry_delay = GCC_get_metrics (cc)->aged_latency;
+    else
+      crm->retry_delay = ch->retry_time;
+  }
+  crm->retry_delay = GNUNET_TIME_STD_BACKOFF (crm->retry_delay);
+  crm->retry_delay = GNUNET_TIME_relative_max (crm->retry_delay,
+                                               MIN_RTT_DELAY);
+  crm->next_retry = GNUNET_TIME_relative_to_absolute (crm->retry_delay);
+
+  GNUNET_CONTAINER_DLL_insert_sorted (struct CadetReliableMessage,
+                                      cmp_crm_by_next_retry,
+                                      NULL,
+                                      ch->head_sent,
+                                      ch->tail_sent,
+                                      crm);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Message %u sent, next transmission on %s in %s\n",
+       (unsigned int) ntohl (crm->data_message->mid.mid),
+       GCCH_2s (ch),
+       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (ch->head_sent->next_retry),
+                                               GNUNET_YES));
+  if (NULL == ch->head_sent->qe)
+  {
+    if (NULL != ch->retry_data_task)
+      GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task
+      = GNUNET_SCHEDULER_add_at (ch->head_sent->next_retry,
+                                 &retry_transmission,
+                                 ch);
+  }
 }
 
 
 /**
- * Handle a channel create requested by a client.
- *
- * Create the channel and the tunnel in case this was the first0 channel.
+ * Handle data given by a client.
  *
- * @param c Client that requested the creation (will be the root).
- * @param msg Create Channel message.
+ * Check whether the client is allowed to send in this tunnel, save if
+ * channel is reliable and send an ACK to the client if there is still
+ * buffer space in the tunnel.
  *
- * @return #GNUNET_OK if everything went fine, #GNUNET_SYSERR otherwise.
+ * @param ch Channel.
+ * @param sender_ccn ccn of the sender
+ * @param buf payload to transmit.
+ * @param buf_len number of bytes in @a buf
+ * @return #GNUNET_OK if everything goes well,
+ *         #GNUNET_SYSERR in case of an error.
  */
 int
-GCCH_handle_local_create (struct CadetClient *c,
-                          struct GNUNET_CADET_LocalChannelCreateMessage *msg)
+GCCH_handle_local_data (struct CadetChannel *ch,
+                        struct GNUNET_CADET_ClientChannelNumber sender_ccn,
+                        const char *buf,
+                        size_t buf_len)
 {
-  struct CadetChannel *ch;
-  struct CadetTunnel *t;
-  struct CadetPeer *peer;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  towards %s:%u\n",
-       GNUNET_i2s (&msg->peer), GNUNET_h2s (&msg->port));
-  ccn = msg->ccn;
-
-  /* Sanity check for duplicate channel IDs */
-  if (NULL != GML_channel_get (c, ccn))
-  {
-    GNUNET_break (0);
-    return GNUNET_SYSERR;
-  }
+  struct CadetReliableMessage *crm;
 
-  peer = GCP_get (&msg->peer, GNUNET_YES);
-  GCP_add_tunnel (peer);
-  t = GCP_get_tunnel (peer);
-
-  if (GCP_get_short_id (peer) == myid)
-  {
-    GCT_change_cstate (t, CADET_TUNNEL_READY);
-  }
-  else
-  {
-    /* FIXME change to a tunnel API, eliminate ch <-> peer connection */
-    GCP_connect (peer);
-  }
-
-  /* Create channel */
-  ch = channel_new (t, c, ccn);
-  if (NULL == ch)
+  if (ch->pending_messages > ch->max_pending_messages)
   {
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
-  ch->port = msg->port;
-  channel_set_options (ch, ntohl (msg->opt));
-
-  /* In unreliable channels, we'll use the DLL to buffer BCK data */
-  ch->root_rel = GNUNET_new (struct CadetChannelReliability);
-  ch->root_rel->ch = ch;
-  ch->root_rel->retry_timer = CADET_RETRANSMIT_TIME;
-  ch->root_rel->expected_delay.rel_value_us = 0;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "CREATED CHANNEL %s\n", GCCH_2s (ch));
-
-  send_create (ch);
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Handler for cadet network payload traffic.
- *
- * @param ch Channel for the message.
- * @param msg Unencryted data message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-void
-GCCH_handle_data (struct CadetChannel *ch,
-                  const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-                  int fwd)
-{
-  struct CadetChannelReliability *rel;
-  struct CadetClient *c;
-  struct GNUNET_MessageHeader *payload_msg;
-  uint32_t mid;
-  uint16_t payload_type;
-  uint16_t payload_size;
-
-  /* If this is a remote (non-loopback) channel, find 'fwd'. */
-  if (GNUNET_SYSERR == fwd)
+  if (GNUNET_YES == ch->destroy)
   {
-    if (is_loopback (ch))
-    {
-      /* It is a loopback channel after all... */
-      GNUNET_break (0);
-      return;
-    }
-    fwd = (NULL != ch->dest) ? GNUNET_YES : GNUNET_NO;
+    /* we are going down, drop messages */
+    return GNUNET_OK;
   }
+  ch->pending_messages++;
 
-  /*  Initialize FWD/BCK data */
-  c   = fwd ? ch->dest     : ch->root;
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-
-  if (NULL == c)
+  if (GNUNET_YES == ch->is_loopback)
   {
-    GNUNET_break (GNUNET_NO != ch->destroy);
-    return;
-  }
+    struct CadetChannelClient *receiver;
+    struct GNUNET_MQ_Envelope *env;
+    struct GNUNET_CADET_LocalData *ld;
+    int ack_to_owner;
 
-  if (CADET_CHANNEL_READY != ch->state)
-  {
-    if (GNUNET_NO == fwd)
+    env = GNUNET_MQ_msg_extra (ld,
+                               buf_len,
+                               GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
+    if ( (NULL != ch->owner) &&
+         (sender_ccn.channel_of_client ==
+          ch->owner->ccn.channel_of_client) )
     {
-      /* If we are the root, this means the other peer has sent traffic before
-       * receiving our ACK. Even if the SYNACK goes missing, no traffic should
-       * be sent before the ACK.
-       */
-      GNUNET_break_op (0);
-      return;
+      receiver = ch->dest;
+      ack_to_owner = GNUNET_YES;
     }
-    /* If we are the dest, this means that the SYNACK got to the root but
-     * the ACK went missing. Treat this as an ACK.
-     */
-    channel_confirm (ch, GNUNET_NO);
-  }
-
-  payload_msg = (struct GNUNET_MessageHeader *) &msg[1];
-  payload_type = ntohs (payload_msg->type);
-  payload_size = ntohs (payload_msg->size);
-
-  GNUNET_STATISTICS_update (stats, "# messages received", 1, GNUNET_NO);
-  GNUNET_STATISTICS_update (stats, "# bytes received", payload_size, GNUNET_NO);
-
-  mid = ntohl (msg->mid);
-  LOG (GNUNET_ERROR_TYPE_INFO, "<== %s (%s %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA), GC_m2s (payload_type), mid,
-       GCCH_2s (ch), ch, GC_f2s (fwd), ntohs (msg->header.size));
-
-  if ( (GNUNET_NO == ch->reliable) ||
-       ( (! GC_is_pid_bigger (rel->mid_recv, mid)) &&
-         GC_is_pid_bigger (rel->mid_recv + 64, mid) ) )
-  {
-    if (GNUNET_YES == ch->reliable)
+    else if ( (NULL != ch->dest) &&
+              (sender_ccn.channel_of_client ==
+               ch->dest->ccn.channel_of_client) )
     {
-      /* Is this the exact next expected messasge? */
-      if (mid == rel->mid_recv)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "as expected, sending to client\n");
-        send_client_data (ch, msg, fwd);
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "save for later\n");
-        add_buffered_data (msg, rel);
-      }
+      receiver = ch->owner;
+      ack_to_owner = GNUNET_NO;
     }
     else
     {
-      /* Tunnel is unreliable: send to clients directly */
-      /* FIXME: accept Out Of Order traffic */
-      rel->mid_recv = mid + 1;
-      send_client_data (ch, msg, fwd);
+      GNUNET_break (0);
+      return GNUNET_SYSERR;
     }
-  }
-  else
-  {
-    GNUNET_STATISTICS_update (stats, "# duplicate MID", 1, GNUNET_NO);
-    if (GC_is_pid_bigger (rel->mid_recv, mid))
+    GNUNET_assert (NULL != receiver);
+    ld->ccn = receiver->ccn;
+    GNUNET_memcpy (&ld[1],
+                   buf,
+                   buf_len);
+    if (GNUNET_YES == receiver->client_ready)
     {
-      GNUNET_break_op (0);
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           "MID %u on channel %s not expected (window: %u - %u). Dropping!\n",
-           mid, GCCH_2s (ch), rel->mid_recv, rel->mid_recv + 63);
+      ch->pending_messages--;
+      GSC_send_to_client (receiver->c,
+                          env);
+      send_ack_to_client (ch,
+                          ack_to_owner);
     }
     else
     {
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           "Duplicate MID %u, channel %s (expecting MID %u). Re-sending ACK!\n",
-           mid, GCCH_2s (ch), rel->mid_recv);
-      if (NULL != rel->uniq)
-      {
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             "We are trying to send an ACK, but don't seem have the "
-             "bandwidth. Have you set enough [ats] QUOTA in your config?\n");
-      }
-
+      struct CadetOutOfOrderMessage *oom;
+
+      oom = GNUNET_new (struct CadetOutOfOrderMessage);
+      oom->env = env;
+      GNUNET_CONTAINER_DLL_insert_tail (receiver->head_recv,
+                                        receiver->tail_recv,
+                                        oom);
+      receiver->num_recv++;
     }
-  }
-
-  GCCH_send_data_ack (ch, fwd);
+    return GNUNET_OK;
+  }
+
+  /* Everything is correct, send the message. */
+  crm = GNUNET_malloc (sizeof (*crm));
+  crm->ch = ch;
+  crm->data_message = GNUNET_malloc (sizeof (struct GNUNET_CADET_ChannelAppDataMessage)
+                                     + buf_len);
+  crm->data_message->header.size = htons (sizeof (struct GNUNET_CADET_ChannelAppDataMessage) + buf_len);
+  crm->data_message->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA);
+  ch->mid_send.mid = htonl (ntohl (ch->mid_send.mid) + 1);
+  crm->data_message->mid = ch->mid_send;
+  crm->data_message->ctn = ch->ctn;
+  GNUNET_memcpy (&crm->data_message[1],
+                 buf,
+                 buf_len);
+  GNUNET_CONTAINER_DLL_insert_tail (ch->head_sent,
+                                    ch->tail_sent,
+                                    crm);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending message %u from local client to %s with %u bytes\n",
+       ntohl (crm->data_message->mid.mid),
+       GCCH_2s (ch),
+       buf_len);
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
+  crm->qe = GCT_send (ch->t,
+                      &crm->data_message->header,
+                      &data_sent_cb,
+                      crm);
+  GNUNET_assert (NULL == ch->retry_data_task);
+  return GNUNET_OK;
 }
 
 
 /**
- * Handler for cadet network traffic end-to-end ACKs.
+ * Handle ACK from client on local channel.  Means the client is ready
+ * for more data, see if we have any for it.
  *
- * @param ch Channel on which we got this message.
- * @param msg Data message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * @param ch channel to destroy
+ * @param client_ccn ccn of the client sending the ack
  */
 void
-GCCH_handle_data_ack (struct CadetChannel *ch,
-                      const struct GNUNET_CADET_ChannelDataAckMessage *msg,
-                      int fwd)
+GCCH_handle_local_ack (struct CadetChannel *ch,
+                       struct GNUNET_CADET_ClientChannelNumber client_ccn)
 {
-  struct CadetChannelReliability *rel;
-  struct CadetReliableMessage *copy;
-  struct CadetReliableMessage *next;
-  uint32_t ack;
-  int work;
-
-  /* If this is a remote (non-loopback) channel, find 'fwd'. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    if (is_loopback (ch))
-    {
-      /* It is a loopback channel after all... */
-      GNUNET_break (0);
-      return;
-    }
-    /* Inverted: if message came 'FWD' is a 'BCK ACK'. */
-    fwd = (NULL != ch->dest) ? GNUNET_NO : GNUNET_YES;
-  }
-
-  ack = ntohl (msg->mid);
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<== %s (0x%010lX %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK), msg->futures, ack,
-       GCCH_2s (ch), ch, GC_f2s (fwd), ntohs (msg->header.size));
-
-  if (GNUNET_YES == fwd)
-    rel = ch->root_rel;
+  struct CadetChannelClient *ccc;
+  struct CadetOutOfOrderMessage *com;
+
+  if ( (NULL != ch->owner) &&
+       (ch->owner->ccn.channel_of_client == client_ccn.channel_of_client) )
+    ccc = ch->owner;
+  else if ( (NULL != ch->dest) &&
+            (ch->dest->ccn.channel_of_client == client_ccn.channel_of_client) )
+    ccc = ch->dest;
   else
-    rel = ch->dest_rel;
-
-  if (NULL == rel)
+    GNUNET_assert (0);
+  ccc->client_ready = GNUNET_YES;
+  com = ccc->head_recv;
+  if (NULL == com)
   {
-    GNUNET_break (GNUNET_NO != ch->destroy);
-    return;
-  }
-
-  /* Free ACK'd copies: no need to retransmit those anymore FIXME refactor */
-  for (work = GNUNET_NO, copy = rel->head_sent; copy != NULL; copy = next)
-  {
-    if (GC_is_pid_bigger (copy->mid, ack))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  head %u, out!\n", copy->mid);
-      if (0 < channel_rel_free_sent (rel, msg))
-        work = GNUNET_YES;
-      break;
-    }
-    work = GNUNET_YES;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  id %u\n", copy->mid);
-    next = copy->next;
-    if (GNUNET_YES == rel_message_free (copy, GNUNET_YES))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " channel destoyed\n");
-      return;
-    }
-  }
-
-  /* ACK client if needed and possible */
-  GCCH_allow_client (ch, fwd);
-
-  /* If some message was free'd, update the retransmission delay */
-  if (GNUNET_YES == work)
-  {
-    if (NULL != rel->retry_task)
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got LOCAL_ACK, %s-%X ready to receive more data, but none pending on %s-%X(%p)!\n",
+         GSC_2s (ccc->c),
+         ntohl (client_ccn.channel_of_client),
+         GCCH_2s (ch),
+         ntohl (ccc->ccn.channel_of_client),
+         ccc);
+    return; /* none pending */
+  }
+  if (GNUNET_YES == ch->is_loopback)
+  {
+    int to_owner;
+
+    /* Messages are always in-order, just send */
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                                 ccc->tail_recv,
+                                 com);
+    ccc->num_recv--;
+    GSC_send_to_client (ccc->c,
+                        com->env);
+    /* Notify sender that we can receive more */
+    if ( (NULL != ch->owner) &&
+         (ccc->ccn.channel_of_client ==
+          ch->owner->ccn.channel_of_client) )
     {
-      GNUNET_SCHEDULER_cancel (rel->retry_task);
-      rel->retry_task = NULL;
-      if (NULL != rel->head_sent && NULL == rel->head_sent->chq)
-      {
-        struct GNUNET_TIME_Absolute new_target;
-        struct GNUNET_TIME_Relative delay;
-
-        delay = GNUNET_TIME_relative_saturating_multiply (rel->retry_timer,
-                                                          CADET_RETRANSMIT_MARGIN);
-        new_target = GNUNET_TIME_absolute_add (rel->head_sent->timestamp,
-                                               delay);
-        delay = GNUNET_TIME_absolute_get_remaining (new_target);
-        rel->retry_task =
-            GNUNET_SCHEDULER_add_delayed (delay,
-                                          &channel_retransmit_message,
-                                          rel);
-      }
+      to_owner = GNUNET_NO;
     }
     else
     {
-      /* Work was done but no task was pending.
-       * Task was cancelled by a retransmission that is sitting in the queue.
-       */
-      // FIXME add test to make sure this is the case, probably add return
-      // value to GCCH_send_prebuilt_message
+      GNUNET_assert ( (NULL != ch->dest) &&
+                      (ccc->ccn.channel_of_client ==
+                       ch->dest->ccn.channel_of_client) );
+      to_owner = GNUNET_YES;
     }
+    send_ack_to_client (ch,
+                        to_owner);
+    GNUNET_free (com);
+    return;
   }
-}
 
-
-/**
- * Handler for channel create messages.
- *
- * Does not have fwd parameter because it's always 'FWD': channel is incoming.
- *
- * @param t Tunnel this channel will be in.
- * @param msg Channel crate message.
- */
-struct CadetChannel *
-GCCH_handle_create (struct CadetTunnel *t,
-                    const struct GNUNET_CADET_ChannelOpenMessage *msg)
-{
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-  struct GNUNET_CADET_ChannelTunnelNumber gid;
-  struct CadetChannel *ch;
-  struct CadetClient *c;
-  int new_channel;
-  const struct GNUNET_HashCode *port;
-
-  gid = msg->ctn;
-  ch = GCT_get_channel (t, gid);
-  if (NULL == ch)
-  {
-    /* Create channel */
-    ccn.channel_of_client = htonl (0);
-    ch = channel_new (t, NULL, ccn);
-    ch->gid = gid;
-    channel_set_options (ch, ntohl (msg->opt));
-    new_channel = GNUNET_YES;
-  }
-  else
+  if ( (com->mid.mid != ch->mid_recv.mid) &&
+       (GNUNET_NO == ch->out_of_order) &&
+       (GNUNET_YES == ch->reliable) )
   {
-    new_channel = GNUNET_NO;
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got LOCAL_ACK, %s-%X ready to receive more data (but next one is out-of-order %u vs. %u)!\n",
+         GSC_2s (ccc->c),
+         ntohl (ccc->ccn.channel_of_client),
+         ntohl (com->mid.mid),
+         ntohl (ch->mid_recv.mid));
+    return; /* missing next one in-order */
   }
-  port = &msg->port;
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<== %s (  0x%08X %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN), ccn, port,
-       GCCH_2s (ch), ch, GC_f2s (GNUNET_YES), ntohs (msg->header.size));
-
-  if (GNUNET_YES == new_channel || GCT_is_loopback (t))
-  {
-    /* Find a destination client */
-    ch->port = *port;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "   port %s\n", GNUNET_h2s (port));
-    c = GML_client_get_by_port (port);
-    if (NULL == c)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  no client has port registered\n");
-      if (is_loopback (ch))
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  loopback: destroy on handler\n");
-        send_nack (ch);
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  not loopback: destroy now\n");
-        send_nack (ch);
-        GCCH_destroy (ch);
-      }
-      return NULL;
-    }
-    else
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  client %p has port registered\n", c);
-    }
-
-    add_destination (ch, c);
-    if (GNUNET_YES == ch->reliable)
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "Reliable\n");
-    else
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "Not Reliable\n");
 
-    send_client_create (ch);
-    ch->state =  CADET_CHANNEL_SENT;
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  duplicate create channel\n");
-    if (NULL != ch->dest_rel->retry_task)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  clearing retry task\n");
-      /* we were waiting to re-send our 'SYNACK', wait no more! */
-      GNUNET_SCHEDULER_cancel (ch->dest_rel->retry_task);
-      ch->dest_rel->retry_task = NULL;
-    }
-    else if (NULL != ch->dest_rel->uniq)
-    {
-      /* we are waiting to for our 'SYNACK' to leave the queue, all done! */
-      return ch;
-    }
-  }
-  send_ack (ch, GNUNET_YES);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got LOCAL_ACK, giving payload message %u to %s-%X on %s\n",
+       ntohl (com->mid.mid),
+       GSC_2s (ccc->c),
+       ntohl (ccc->ccn.channel_of_client),
+       GCCH_2s (ch));
 
-  return ch;
+  /* all good, pass next message to client */
+  GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                               ccc->tail_recv,
+                               com);
+  ccc->num_recv--;
+  /* FIXME: if unreliable, this is not aggressive
+     enough, as it would be OK to have lost some! */
+
+  ch->mid_recv.mid = htonl (1 + ntohl (com->mid.mid));
+  ch->mid_futures >>= 1; /* equivalent to division by 2 */
+  ccc->client_ready = GNUNET_NO;
+  GSC_send_to_client (ccc->c,
+                      com->env);
+  GNUNET_free (com);
+  send_channel_data_ack (ch);
+  if (NULL != ccc->head_recv)
+    return;
+  if (GNUNET_NO == ch->destroy)
+    return;
+  GCT_send_channel_destroy (ch->t,
+                            ch->ctn);
+  channel_destroy (ch);
 }
 
 
-/**
- * Handler for channel NACK messages.
- *
- * NACK messages always go dest -> root, no need for 'fwd' or 'msg' parameter.
- *
- * @param ch Channel.
- */
-void
-GCCH_handle_nack (struct CadetChannel *ch)
-{
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<== %s (  0x%08X %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED), ch->gid, 0,
-       GCCH_2s (ch), ch, "---", 0);
-
-  send_client_nack (ch);
-  GCCH_destroy (ch);
-}
+#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-chn",__VA_ARGS__)
 
 
 /**
- * Handler for channel ack messages.
+ * Log channel info.
  *
  * @param ch Channel.
- * @param msg Message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-void
-GCCH_handle_ack (struct CadetChannel *ch,
-                 const struct GNUNET_CADET_ChannelManageMessage *msg,
-                 int fwd)
-{
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<== %s (  0x%08X %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK), ch->gid, 0,
-       GCCH_2s (ch), ch, GC_f2s (fwd), ntohs (msg->header.size));
-
-  /* If this is a remote (non-loopback) channel, find 'fwd'. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    if (is_loopback (ch))
-    {
-      /* It is a loopback channel after all... */
-      GNUNET_break (0);
-      return;
-    }
-    fwd = (NULL != ch->dest) ? GNUNET_YES : GNUNET_NO;
-  }
-
-  channel_confirm (ch, !fwd);
-}
-
-
-/**
- * Handler for channel destroy messages.
- *
- * @param ch Channel to be destroyed of.
- * @param msg Message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * @param level Debug level to use.
  */
 void
-GCCH_handle_destroy (struct CadetChannel *ch,
-                     const struct GNUNET_CADET_ChannelManageMessage *msg,
-                     int fwd)
+GCCH_debug (struct CadetChannel *ch,
+            enum GNUNET_ErrorType level)
 {
-  struct CadetChannelReliability *rel;
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<== %s (  0x%08X %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY), ch->gid, 0,
-       GCCH_2s (ch), ch, GC_f2s (fwd), ntohs (msg->header.size));
-
-  /* If this is a remote (non-loopback) channel, find 'fwd'. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    if (is_loopback (ch))
-    {
-      /* It is a loopback channel after all... */
-      GNUNET_break (0);
-      return;
-    }
-    fwd = (NULL != ch->dest) ? GNUNET_YES : GNUNET_NO;
-  }
+  int do_log;
 
-  GCCH_debug (ch, GNUNET_ERROR_TYPE_DEBUG);
-  if ( (fwd && NULL == ch->dest) || (!fwd && NULL == ch->root) )
-  {
-    /* Not for us (don't destroy twice a half-open loopback channel) */
+  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
+                                       "cadet-chn",
+                                       __FILE__, __FUNCTION__, __LINE__);
+  if (0 == do_log)
     return;
-  }
-
-  rel = fwd ? ch->dest_rel : ch->root_rel;
-  if (0 == rel->n_recv)
-  {
-    send_destroy (ch, GNUNET_YES);
-    GCCH_destroy (ch);
-  }
-  else
-  {
-    ch->destroy = GNUNET_YES;
-  }
-}
-
-
-/**
- * Sends an already built message on a channel.
- *
- * If the channel is on a loopback tunnel, notifies the appropriate destination
- * client locally.
- *
- * On a normal channel passes the message to the tunnel for encryption and
- * sending on a connection.
- *
- * This function DOES NOT save the message for retransmission.
- *
- * @param message Message to send. Function makes a copy of it.
- * @param ch Channel on which this message is transmitted.
- * @param fwd Is this a fwd message?
- * @param existing_copy This is a retransmission, don't save a copy.
- */
-void
-GCCH_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                            struct CadetChannel *ch, int fwd,
-                            void *existing_copy)
-{
-  struct CadetChannelQueue *chq;
-  uint32_t data_id;
-  uint16_t type;
-  uint16_t size;
-  char info[32];
-
-  type = ntohs (message->type);
-  size = ntohs (message->size);
-
-  data_id = 0;
-  switch (type)
-  {
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-    {
-      struct GNUNET_CADET_ChannelAppDataMessage *data_msg;
-      struct GNUNET_MessageHeader *payload_msg;
-      uint16_t payload_type;
-
-      data_msg = (struct GNUNET_CADET_ChannelAppDataMessage *) message;
-      data_id = ntohl (data_msg->mid);
-      payload_msg = (struct GNUNET_MessageHeader *) &data_msg[1];
-      payload_type = ntohs (payload_msg->type);
-      strncpy (info, GC_m2s (payload_type), 31);
-      info[31] = '\0';
-      break;
-    }
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-    {
-      struct GNUNET_CADET_ChannelDataAckMessage *ack_msg;
-      ack_msg = (struct GNUNET_CADET_ChannelDataAckMessage *) message;
-      data_id = ntohl (ack_msg->mid);
-      SPRINTF (info, "0x%010lX",
-               (unsigned long int) ack_msg->futures);
-      break;
-    }
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-    {
-      struct GNUNET_CADET_ChannelOpenMessage *cc_msg;
-      cc_msg = (struct GNUNET_CADET_ChannelOpenMessage *) message;
-      SPRINTF (info, "  0x%08X", ntohl (cc_msg->ctn.cn));
-      break;
-    }
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-    {
-      struct GNUNET_CADET_ChannelManageMessage *m_msg;
-      m_msg = (struct GNUNET_CADET_ChannelManageMessage *) message;
-      SPRINTF (info, "  0x%08X", ntohl (m_msg->ctn.cn));
-      break;
-    }
-    default:
-      info[0] = '\0';
-  }
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "==> %s (%12s %4u) on chan %s (%p) %s [%5u]\n",
-       GC_m2s (type), info, data_id,
-       GCCH_2s (ch), ch, GC_f2s (fwd), size);
 
-  if (GCT_is_loopback (ch->t))
+  if (NULL == ch)
   {
-    handle_loopback (ch, message, fwd);
+    LOG2 (level, "CHN *** DEBUG NULL CHANNEL ***\n");
     return;
   }
-
-  switch (type)
+  LOG2 (level,
+        "CHN %s:%X (%p)\n",
+        GCT_2s (ch->t),
+        ch->ctn,
+        ch);
+  if (NULL != ch->owner)
   {
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-      if (GNUNET_YES == ch->reliable)
-      {
-        chq = GNUNET_new (struct CadetChannelQueue);
-        chq->type = type;
-        if (NULL == existing_copy)
-          chq->copy = channel_save_copy (ch, message, fwd);
-        else
-        {
-          chq->copy = (struct CadetReliableMessage *) existing_copy;
-          if (NULL != chq->copy->chq)
-          {
-            /* Last retransmission was queued but not yet sent!
-             * This retransmission was scheduled by a ch_message_sent which
-             * followed a very fast RTT, so the tiny delay made the
-             * retransmission function to execute before the previous
-             * retransmitted message even had a chance to leave the peer.
-             * Cancel this message and wait until the pending
-             * retransmission leaves the peer and ch_message_sent starts
-             * the timer for the next one.
-             */
-            GNUNET_free (chq);
-            LOG (GNUNET_ERROR_TYPE_DEBUG,
-                 "  exisitng copy not yet transmitted!\n");
-            return;
-          }
-          LOG (GNUNET_ERROR_TYPE_DEBUG,
-               "  using existing copy: %p {r:%p q:%p t:%u}\n",
-               existing_copy,
-               chq->copy->rel, chq->copy->chq, chq->copy->type);
-        }
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  new chq: %p\n", chq);
-        chq->copy->chq = chq;
-        chq->tq = GCT_send_prebuilt_message (message, ch->t, NULL,
-                                             GNUNET_YES,
-                                             &ch_message_sent, chq);
-        /* q itself is stored in copy */
-        GNUNET_assert (NULL != chq->tq || GNUNET_NO != ch->destroy);
-      }
-      else
-      {
-        fire_and_forget (message, ch, GNUNET_NO);
-      }
-      break;
-
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-      chq = GNUNET_new (struct CadetChannelQueue);
-      chq->type = type;
-      chq->rel = fwd ? ch->root_rel : ch->dest_rel;
-      if (NULL != chq->rel->uniq)
-      {
-        if (NULL != chq->rel->uniq->tq)
-        {
-          GCT_cancel (chq->rel->uniq->tq);
-          /* ch_message_sent is called, freeing and NULLing uniq */
-          GNUNET_break (NULL == chq->rel->uniq);
-        }
-        else
-        {
-          GNUNET_break (0);
-          GNUNET_free (chq->rel->uniq);
-        }
-      }
-
-      chq->rel->uniq = chq;
-      chq->tq = GCT_send_prebuilt_message (message, ch->t, NULL, GNUNET_YES,
-                                           &ch_message_sent, chq);
-      if (NULL == chq->tq)
-      {
-        GNUNET_break (0);
-        chq->rel->uniq = NULL;
-        GCT_debug (ch->t, GNUNET_ERROR_TYPE_ERROR);
-        GNUNET_free (chq);
-        chq = NULL;
-        return;
-      }
-      break;
-
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED:
-      fire_and_forget (message, ch, GNUNET_YES);
-      break;
-
-
-    default:
-      GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_WARNING, "type %s unknown!\n", GC_m2s (type));
-      fire_and_forget (message, ch, GNUNET_YES);
+    LOG2 (level,
+          "CHN origin %s ready %s local-id: %u\n",
+          GSC_2s (ch->owner->c),
+          ch->owner->client_ready ? "YES" : "NO",
+          ntohl (ch->owner->ccn.channel_of_client));
   }
+  if (NULL != ch->dest)
+  {
+    LOG2 (level,
+          "CHN destination %s ready %s local-id: %u\n",
+          GSC_2s (ch->dest->c),
+          ch->dest->client_ready ? "YES" : "NO",
+          ntohl (ch->dest->ccn.channel_of_client));
+  }
+  LOG2 (level,
+        "CHN  Message IDs recv: %d (%LLX), send: %d\n",
+        ntohl (ch->mid_recv.mid),
+        (unsigned long long) ch->mid_futures,
+        ntohl (ch->mid_send.mid));
 }
 
 
-/**
- * Get the static string for identification of the channel.
- *
- * @param ch Channel.
- *
- * @return Static string with the channel IDs.
- */
-const char *
-GCCH_2s (const struct CadetChannel *ch)
-{
-  static char buf[64];
-
-  if (NULL == ch)
-    return "(NULL Channel)";
 
-  SPRINTF (buf,
-           "%s:%s gid:%X (%X / %X)",
-           GCT_2s (ch->t),
-           GNUNET_h2s (&ch->port),
-           ntohl (ch->gid.cn),
-           ntohl (ch->lid_root.channel_of_client),
-           ntohl (ch->lid_dest.channel_of_client));
-
-  return buf;
-}
+/* end of gnunet-service-cadet-new_channel.c */
diff --git a/src/cadet/gnunet-service-cadet_channel.h b/src/cadet/gnunet-service-cadet_channel.h
index 9d4893269..a3ef9a06d 100644
--- a/src/cadet/gnunet-service-cadet_channel.h
+++ b/src/cadet/gnunet-service-cadet_channel.h
@@ -1,6 +1,7 @@
+
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -20,344 +21,242 @@
 
 /**
  * @file cadet/gnunet-service-cadet_channel.h
- * @brief cadet service; dealing with end-to-end channels
+ * @brief GNUnet CADET service with encryption
  * @author Bartlomiej Polot
- *
- * All functions in this file should use the prefix GMCH (Gnunet Cadet CHannel)
+ * @author Christian Grothoff
  */
-
 #ifndef GNUNET_SERVICE_CADET_CHANNEL_H
 #define GNUNET_SERVICE_CADET_CHANNEL_H
 
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
+#include "gnunet-service-cadet.h"
+#include "gnunet-service-cadet_peer.h"
 #include "cadet_protocol.h"
-#include "cadet.h"
-
-/**
- * Struct containing all information regarding a channel to a remote client.
- */
-struct CadetChannel;
-
-
-#include "gnunet-service-cadet_tunnel.h"
-#include "gnunet-service-cadet_local.h"
-
-
-/**
- * Destroy a channel and free all resources.
- *
- * @param ch Channel to destroy.
- */
-void
-GCCH_destroy (struct CadetChannel *ch);
 
 
 /**
- * Get the channel's public ID.
- *
- * @param ch Channel.
- *
- * @return ID used to identify the channel with the remote peer.
+ * A channel is a bidirectional connection between two CADET
+ * clients.  Communiation can be reliable, unreliable, in-order
+ * or out-of-order.  One client is the "local" client, this
+ * one initiated the connection.   The other client is the
+ * "incoming" client, this one listened on a port to accept
+ * the connection from the "local" client.
  */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCCH_get_id (const struct CadetChannel *ch);
-
-
-/**
- * Get the channel tunnel.
- *
- * @param ch Channel to get the tunnel from.
- *
- * @return tunnel of the channel.
- */
-struct CadetTunnel *
-GCCH_get_tunnel (const struct CadetChannel *ch);
+struct CadetChannel;
 
 
 /**
- * Get free buffer space towards the client on a specific channel.
+ * Get the static string for identification of the channel.
  *
  * @param ch Channel.
- * @param fwd Is query about FWD traffic?
  *
- * @return Free buffer space [0 - 64]
+ * @return Static string with the channel IDs.
  */
-unsigned int
-GCCH_get_buffer (struct CadetChannel *ch, int fwd);
+const char *
+GCCH_2s (const struct CadetChannel *ch);
 
 
 /**
- * Get flow control status of end point: is client allow to send?
+ * Log channel info.
  *
  * @param ch Channel.
- * @param fwd Is query about FWD traffic? (Request root status).
- *
- * @return #GNUNET_YES if client is allowed to send us data.
+ * @param level Debug level to use.
  */
-int
-GCCH_get_allowed (struct CadetChannel *ch, int fwd);
+void
+GCCH_debug (struct CadetChannel *ch,
+            enum GNUNET_ErrorType level);
 
 
 /**
- * Is the root client for this channel on this peer?
+ * Get the channel's public ID.
  *
  * @param ch Channel.
- * @param fwd Is this for fwd traffic?
  *
- * @return #GNUNET_YES in case it is.
+ * @return ID used to identify the channel with the remote peer.
  */
-int
-GCCH_is_origin (struct CadetChannel *ch, int fwd);
+struct GNUNET_CADET_ChannelTunnelNumber
+GCCH_get_id (const struct CadetChannel *ch);
 
-/**
- * Is the destination client for this channel on this peer?
- *
- * @param ch Channel.
- * @param fwd Is this for fwd traffic?
- *
- * @return #GNUNET_YES in case it is.
- */
-int
-GCCH_is_terminal (struct CadetChannel *ch, int fwd);
 
 /**
- * Send an end-to-end ACK message for the most recent in-sequence payload.
- *
- * If channel is not reliable, do nothing.
- *
- * @param ch Channel this is about.
- * @param fwd Is for FWD traffic? (ACK dest->owner)
+ * Create a new channel.
+ *
+ * @param owner local client owning the channel
+ * @param owner_id local chid of this channel at the @a owner
+ * @param destination peer to which we should build the channel
+ * @param port desired port at @a destination
+ * @param options options for the channel
+ * @return handle to the new channel
  */
-void
-GCCH_send_data_ack (struct CadetChannel *ch, int fwd);
+struct CadetChannel *
+GCCH_channel_local_new (struct CadetClient *owner,
+                        struct GNUNET_CADET_ClientChannelNumber owner_id,
+                        struct CadetPeer *destination,
+                        const struct GNUNET_HashCode *port,
+                        uint32_t options);
 
-/**
- * Notify the destination client that a new incoming channel was created.
- *
- * @param ch Channel that was created.
- */
-void
-GCCH_send_create (struct CadetChannel *ch);
 
 /**
- * Allow a client to send us more data, in case it was choked.
+ * A client is bound to the port that we have a channel
+ * open to.  Send the acknowledgement for the connection
+ * request and establish the link with the client.
  *
- * @param ch Channel.
- * @param fwd Is this about FWD traffic? (Root client).
+ * @param ch open incoming channel
+ * @param c client listening on the respective port
  */
 void
-GCCH_allow_client (struct CadetChannel *ch, int fwd);
+GCCH_bind (struct CadetChannel *ch,
+           struct CadetClient *c);
 
-/**
- * Log channel info.
- *
- * @param ch Channel.
- * @param level Debug level to use.
- */
-void
-GCCH_debug (struct CadetChannel *ch, enum GNUNET_ErrorType level);
 
 /**
- * Handle an ACK given by a client.
+ * Destroy locally created channel.  Called by the
+ * local client, so no need to tell the client.
  *
- * Mark client as ready and send him any buffered data we could have for him.
- *
- * @param ch Channel.
- * @param fwd Is this a "FWD ACK"? (FWD ACKs are sent by root and go BCK)
+ * @param ch channel to destroy
+ * @param c client that caused the destruction
+ * @param ccn client number of the client @a c
  */
 void
-GCCH_handle_local_ack (struct CadetChannel *ch, int fwd);
+GCCH_channel_local_destroy (struct CadetChannel *ch,
+                            struct CadetClient *c,
+                            struct GNUNET_CADET_ClientChannelNumber ccn);
 
-/**
- * Handle data given by a client.
- *
- * Check whether the client is allowed to send in this tunnel, save if channel
- * is reliable and send an ACK to the client if there is still buffer space
- * in the tunnel.
- *
- * @param ch Channel.
- * @param c Client which sent the data.
- * @param fwd Is this a FWD data?
- * @param message Data message.
- * @param size Size of data.
- *
- * @return GNUNET_OK if everything goes well, GNUNET_SYSERR in case of en error.
- */
-int
-GCCH_handle_local_data (struct CadetChannel *ch,
-                        struct CadetClient *c, int fwd,
-                        const struct GNUNET_MessageHeader *message,
-                        size_t size);
 
 /**
- * Handle a channel destroy requested by a client.
- *
- * Destroy the channel and the tunnel in case this was the last channel.
- *
- * @param ch Channel.
- * @param c Client that requested the destruction (to avoid notifying him).
- * @param is_root Is the request coming from root?
+ * Function called once and only once after a channel was bound
+ * to its tunnel via #GCT_add_channel() is ready for transmission.
+ * Note that this is only the case for channels that this peer
+ * initiates, as for incoming channels we assume that they are
+ * ready for transmission immediately upon receiving the open
+ * message.  Used to bootstrap the #GCT_send() process.
+ *
+ * @param ch the channel for which the tunnel is now ready
  */
 void
-GCCH_handle_local_destroy (struct CadetChannel *ch,
-                           struct CadetClient *c,
-                           int is_root);
+GCCH_tunnel_up (struct CadetChannel *ch);
 
 
 /**
- * Handle a channel create requested by a client.
- *
- * Create the channel and the tunnel in case this was the first0 channel.
- *
- * @param c Client that requested the creation (will be the root).
- * @param msg Create Channel message.
- *
- * @return #GNUNET_OK if everything went fine, #GNUNET_SYSERR otherwise.
+ * Create a new channel based on a request coming in over the network.
+ *
+ * @param t tunnel to the remote peer
+ * @param chid identifier of this channel in the tunnel
+ * @param origin peer to who initiated the channel
+ * @param port desired local port
+ * @param options options for the channel
+ * @return handle to the new channel
  */
-int
-GCCH_handle_local_create (struct CadetClient *c,
-                          struct GNUNET_CADET_LocalChannelCreateMessage *msg);
-
-/**
- * Handler for cadet network payload traffic.
- *
- * @param ch Channel for the message.
- * @param msg Unencryted data message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-void
-GCCH_handle_data (struct CadetChannel *ch,
-                  const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-                  int fwd);
+struct CadetChannel *
+GCCH_channel_incoming_new (struct CadetTunnel *t,
+                           struct GNUNET_CADET_ChannelTunnelNumber chid,
+                           const struct GNUNET_HashCode *port,
+                           uint32_t options);
 
 
 /**
- * Handler for cadet network traffic end-to-end ACKs.
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN message again for
+ * this channel.  If the binding was successful, (re)transmit the
+ * #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK.
  *
- * @param ch Channel on which we got this message.
- * @param msg Data message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * @param ch channel that got the duplicate open
+ * @param cti identifier of the connection that delivered the message
  */
 void
-GCCH_handle_data_ack (struct CadetChannel *ch,
-                      const struct GNUNET_CADET_ChannelDataAckMessage *msg,
-                      int fwd);
+GCCH_handle_duplicate_open (struct CadetChannel *ch,
+                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti);
 
 
-/**
- * Handler for channel create messages.
- *
- * Does not have fwd parameter because it's always 'FWD': channel is incoming.
- *
- * @param t Tunnel this channel will be in.
- * @param msg Channel crate message.
- */
-struct CadetChannel *
-GCCH_handle_create (struct CadetTunnel *t,
-                    const struct GNUNET_CADET_ChannelOpenMessage *msg);
-
 
 /**
- * Handler for channel NACK messages.
+ * We got payload data for a channel.  Pass it on to the client.
  *
- * NACK messages always go dest -> root, no need for 'fwd' or 'msg' parameter.
- *
- * @param ch Channel.
+ * @param ch channel that got data
+ * @param cti identifier of the connection that delivered the message
+ * @param msg message that was received
  */
 void
-GCCH_handle_nack (struct CadetChannel *ch);
+GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
+                                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                                    const struct GNUNET_CADET_ChannelAppDataMessage *msg);
 
 
 /**
- * Handler for channel ack messages.
+ * We got an acknowledgement for payload data for a channel.
+ * Possibly resume transmissions.
  *
- * @param ch Channel this channel is to be created in.
- * @param msg Message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * @param ch channel that got the ack
+ * @param cti identifier of the connection that delivered the message
+ * @param ack details about what was received
  */
 void
-GCCH_handle_ack (struct CadetChannel *ch,
-                 const struct GNUNET_CADET_ChannelManageMessage *msg,
-                 int fwd);
+GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
+                                        const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                                        const struct GNUNET_CADET_ChannelDataAckMessage *ack);
 
 
 /**
- * Handler for channel destroy messages.
+ * We got an acknowledgement for the creation of the channel
+ * (the port is open on the other side). Begin transmissions.
  *
- * @param ch Channel this channel is to be destroyed of.
- * @param msg Message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ * @param ch channel to destroy
+ * @param cti identifier of the connection that delivered the message,
+ *        NULL if the ACK was inferred because we got payload or are on loopback
  */
 void
-GCCH_handle_destroy (struct CadetChannel *ch,
-                     const struct GNUNET_CADET_ChannelManageMessage *msg,
-                     int fwd);
+GCCH_handle_channel_open_ack (struct CadetChannel *ch,
+                              const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti);
 
 
 /**
- * Sends an already built message on a channel.
+ * Destroy channel, based on the other peer closing the
+ * connection.  Also needs to remove this channel from
+ * the tunnel.
  *
- * If the channel is on a loopback tunnel, notifies the appropriate destination
- * client locally.
+ * FIXME: need to make it possible to defer destruction until we have
+ * received all messages up to the destroy, and right now the destroy
+ * message (and this API) fails to give is the information we need!
  *
- * On a normal channel passes the message to the tunnel for encryption and
- * sending on a connection.
+ * FIXME: also need to know if the other peer got a destroy from
+ * us before!
  *
- * This function DOES NOT save the message for retransmission.
- *
- * @param message Message to send. Function makes a copy of it.
- * @param ch Channel on which this message is transmitted.
- * @param fwd Is this a fwd message?
- * @param existing_copy This is a retransmission, don't save a copy.
+ * @param ch channel to destroy
+ * @param cti identifier of the connection that delivered the message,
+ *            NULL during shutdown
  */
 void
-GCCH_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                            struct CadetChannel *ch, int fwd,
-                            void *existing_copy);
+GCCH_handle_remote_destroy (struct CadetChannel *ch,
+                            const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti);
 
 
 /**
- * Get the static string for identification of the channel.
+ * Handle data given by a client.
  *
- * @param ch Channel.i
+ * Check whether the client is allowed to send in this tunnel, save if
+ * channel is reliable and send an ACK to the client if there is still
+ * buffer space in the tunnel.
  *
- * @return Static string with the channel IDs.
+ * @param ch Channel.
+ * @param sender_ccn ccn of the sender
+ * @param buf payload to transmit.
+ * @param buf_len number of bytes in @a buf
+ * @return #GNUNET_OK if everything goes well,
+ *         #GNUNET_SYSERR in case of an error.
  */
-const char *
-GCCH_2s (const struct CadetChannel *ch);
-
-
+int
+GCCH_handle_local_data (struct CadetChannel *ch,
+                        struct GNUNET_CADET_ClientChannelNumber sender_ccn,
+                        const char *buf,
+                        size_t buf_len);
 
 
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
+/**
+ * Handle ACK from client on local channel.
+ *
+ * @param ch channel to destroy
+ * @param client_ccn ccn of the client sending the ack
+ */
+void
+GCCH_handle_local_ack (struct CadetChannel *ch,
+                       struct GNUNET_CADET_ClientChannelNumber client_ccn);
 
-/* ifndef GNUNET_SERVICE_CADET_CHANNEL_H */
 #endif
-/* end of gnunet-service-cadet_channel.h */
diff --git a/src/cadet/gnunet-service-cadet_connection.c b/src/cadet/gnunet-service-cadet_connection.c
index 2d5087f81..7b66f61a2 100644
--- a/src/cadet/gnunet-service-cadet_connection.c
+++ b/src/cadet/gnunet-service-cadet_connection.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2001-2015 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,242 +17,126 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
+
 /**
  * @file cadet/gnunet-service-cadet_connection.c
- * @brief GNUnet CADET service connection handling
+ * @brief management of CORE-level end-to-end connections; establishes
+ *        end-to-end routes and transmits messages along the route
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
  */
 #include "platform.h"
-#include "gnunet_util_lib.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet_paths.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet_cadet_service.h"
 #include "gnunet_statistics_service.h"
-#include "cadet_path.h"
 #include "cadet_protocol.h"
-#include "cadet.h"
-#include "gnunet-service-cadet_connection.h"
-#include "gnunet-service-cadet_peer.h"
-#include "gnunet-service-cadet_tunnel.h"
-
-
-/**
- * Should we run somewhat expensive checks on our invariants?
- */
-#define CHECK_INVARIANTS 0
-
-
-#define LOG(level, ...) GNUNET_log_from (level,"cadet-con",__VA_ARGS__)
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-con",__VA_ARGS__)
-
-
-#define CADET_MAX_POLL_TIME      GNUNET_TIME_relative_multiply (\
-                                  GNUNET_TIME_UNIT_MINUTES,\
-                                  10)
-#define AVG_MSGS                32
-
-
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
-
-/**
- * Handle for messages queued but not yet sent.
- */
-struct CadetConnectionQueue
-{
 
-  struct CadetConnectionQueue *next;
-  struct CadetConnectionQueue *prev;
 
-  /**
-   * Peer queue handle, to cancel if necessary.
-   */
-  struct CadetPeerQueue *peer_q;
-
-  /**
-   * Continuation to call once sent.
-   */
-  GCC_sent cont;
-
-  /**
-   * Closure for @e cont.
-   */
-  void *cont_cls;
-
-  /**
-   * Was this a forced message? (Do not account for it)
-   */
-  int forced;
-};
+#define LOG(level, ...) GNUNET_log_from(level,"cadet-con",__VA_ARGS__)
 
 
 /**
- * Struct to encapsulate all the Flow Control information to a peer to which
- * we are directly connected (on a core level).
+ * All the states a connection can be in.
  */
-struct CadetFlowControl
+enum CadetConnectionState
 {
   /**
-   * Connection this controls.
-   */
-  struct CadetConnection *c;
-
-  struct CadetConnectionQueue *q_head;
-  struct CadetConnectionQueue *q_tail;
-
-  /**
-   * How many messages are in the queue on this connection.
-   */
-  unsigned int queue_n;
-
-  /**
-   * How many messages do we accept in the queue.
-   * If 0, the connection is broken in this direction (next hop disconnected).
-   */
-  unsigned int queue_max;
-
-  /**
-   * ID of the next packet to send.
-   */
-  struct CadetEncryptedMessageIdentifier next_pid;
-
-  /**
-   * ID of the last packet sent towards the peer.
-   */
-  struct CadetEncryptedMessageIdentifier last_pid_sent;
-
-  /**
-   * ID of the last packet received from the peer.
-   */
-  struct CadetEncryptedMessageIdentifier last_pid_recv;
-
-  /**
-   * Bitmap of past 32 messages received:
-   * - LSB being @c last_pid_recv.
-   * - MSB being @c last_pid_recv - 31 (mod UINTMAX).
-   */
-  uint32_t recv_bitmap;
-
-  /**
-   * Last ACK sent to the peer (peer is not allowed to send
-   * messages with PIDs higher than this value).
+   * Uninitialized status, we have not yet even gotten the message queue.
    */
-  struct CadetEncryptedMessageIdentifier last_ack_sent;
+  CADET_CONNECTION_NEW,
 
   /**
-   * Last ACK sent towards the origin (for traffic towards leaf node).
+   * Connection create message in queue, awaiting transmission by CORE.
    */
-  struct CadetEncryptedMessageIdentifier last_ack_recv;
+  CADET_CONNECTION_SENDING_CREATE,
 
   /**
-   * Task to poll the peer in case of a lost ACK causes stall.
+   * Connection create message sent, waiting for ACK.
    */
-  struct GNUNET_SCHEDULER_Task *poll_task;
+  CADET_CONNECTION_SENT,
 
   /**
-   * How frequently to poll for ACKs.
+   * We are an inbound connection, and received a CREATE. Need to
+   * send an CREATE_ACK back.
    */
-  struct GNUNET_TIME_Relative poll_time;
+  CADET_CONNECTION_CREATE_RECEIVED,
 
   /**
-   * Queued poll message, to cancel if not necessary anymore (got ACK).
+   * Connection confirmed, ready to carry traffic.
    */
-  struct CadetConnectionQueue *poll_msg;
+  CADET_CONNECTION_READY
 
-  /**
-   * Queued poll message, to cancel if not necessary anymore (got ACK).
-   */
-  struct CadetConnectionQueue *ack_msg;
 };
 
+
 /**
- * Keep a record of the last messages sent on this connection.
+ * Low-level connection to a destination.
  */
-struct CadetConnectionPerformance
+struct CadetConnection
 {
-  /**
-   * Circular buffer for storing measurements.
-   */
-  double usecsperbyte[AVG_MSGS];
 
   /**
-   * Running average of @c usecsperbyte.
-   */
-  double avg;
-
-  /**
-   * How many values of @c usecsperbyte are valid.
-   */
-  uint16_t size;
-
-  /**
-   * Index of the next "free" position in @c usecsperbyte.
+   * ID of the connection.
    */
-  uint16_t idx;
-};
-
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
 
-/**
- * Struct containing all information regarding a connection to a peer.
- */
-struct CadetConnection
-{
   /**
-   * Tunnel this connection is part of.
+   * To which peer does this connection go?
    */
-  struct CadetTunnel *t;
+  struct CadetPeer *destination;
 
   /**
-   * Flow control information for traffic fwd.
+   * Which tunnel is using this connection?
    */
-  struct CadetFlowControl fwd_fc;
+  struct CadetTConnection *ct;
 
   /**
-   * Flow control information for traffic bck.
+   * Path we are using to our destination.
    */
-  struct CadetFlowControl bck_fc;
+  struct CadetPeerPath *path;
 
   /**
-   * Measure connection performance on the endpoint.
+   * Pending message, NULL if we are ready to transmit.
    */
-  struct CadetConnectionPerformance *perf;
+  struct GNUNET_MQ_Envelope *env;
 
   /**
-   * ID of the connection.
+   * Handle for calling #GCP_request_mq_cancel() once we are finished.
    */
-  struct GNUNET_CADET_ConnectionTunnelIdentifier id;
+  struct GCP_MessageQueueManager *mq_man;
 
   /**
-   * Path being used for the tunnel. At the origin of the connection
-   * it's a pointer to the destination's path pool, otherwise just a copy.
+   * Task for connection maintenance.
    */
-  struct CadetPeerPath *path;
+  struct GNUNET_SCHEDULER_Task *task;
 
   /**
-   * Task to keep the used paths alive at the owner,
-   * time tunnel out on all the other peers.
+   * Queue entry for keepalive messages.
    */
-  struct GNUNET_SCHEDULER_Task *fwd_maintenance_task;
+  struct CadetTunnelQueueEntry *keepalive_qe;
 
   /**
-   * Task to keep the used paths alive at the destination,
-   * time tunnel out on all the other peers.
+   * Function to call once we are ready to transmit.
    */
-  struct GNUNET_SCHEDULER_Task *bck_maintenance_task;
+  GCC_ReadyCallback ready_cb;
 
   /**
-   * Queue handle for maintainance traffic. One handle for FWD and BCK since
-   * one peer never needs to maintain both directions (no loopback connections).
+   * Closure for @e ready_cb.
    */
-  struct CadetPeerQueue *maintenance_q;
+  void *ready_cb_cls;
 
   /**
-   * Should equal #get_next_hop(), or NULL if that peer disconnected.
+   * How long do we wait before we try again with a CREATE message?
    */
-  struct CadetPeer *next_peer;
+  struct GNUNET_TIME_Relative retry_delay;
 
   /**
-   * Should equal #get_prev_hop(), or NULL if that peer disconnected.
+   * Performance metrics for this connection.
    */
-  struct CadetPeer *prev_peer;
+  struct CadetConnectionMetrics metrics;
 
   /**
    * State of the connection.
@@ -260,221 +144,36 @@ struct CadetConnection
   enum CadetConnectionState state;
 
   /**
-   * Position of the local peer in the path.
+   * Options for the route, control buffering.
    */
-  unsigned int own_pos;
+  enum GNUNET_CADET_ChannelOption options;
 
   /**
-   * Pending message count.
+   * How many latency observations did we make for this connection?
    */
-  unsigned int pending_messages;
+  unsigned int latency_datapoints;
 
   /**
-   * Destroy flag:
-   * - if 0, connection in use.
-   * - if 1, destroy on last message.
-   * - if 2, connection is being destroyed don't re-enter.
+   * Offset of our @e destination in @e path.
    */
-  int destroy;
+  unsigned int off;
 
   /**
-   * In-connection-map flag. Sometimes, when @e destroy is set but
-   * actual destruction is delayed to enable us to finish processing
-   * queues (i.e. in the direction that is still working), we remove
-   * the connection from the map to prevent it from still being
-   * found (and used) by accident. This flag is set to #GNUNET_YES
-   * for a connection that is not in the #connections map.  Should
-   * only be #GNUNET_YES if #destroy is also non-zero.
+   * Are we ready to transmit via @e mq_man right now?
    */
-  int was_removed;
+  int mqm_ready;
 
-  /**
-   * Counter to do exponential backoff when creating a connection (max 64).
-   */
-  unsigned short create_retry;
-
-  /**
-   * Task to check if connection has duplicates.
-   */
-  struct GNUNET_SCHEDULER_Task *check_duplicates_task;
 };
 
 
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Local peer own ID (memory efficient handle).
- */
-extern GNUNET_PEER_Id myid;
-
-/**
- * Local peer own ID (full value).
- */
-extern struct GNUNET_PeerIdentity my_full_id;
-
-/**
- * Connections known, indexed by cid (CadetConnection).
- */
-static struct GNUNET_CONTAINER_MultiShortmap *connections;
-
-/**
- * How many connections are we willing to maintain.
- *  Local connections are always allowed,
- * even if there are more connections than max.
- */
-static unsigned long long max_connections;
-
-/**
- * How many messages *in total* are we willing to queue, divide by number of
- * connections to get connection queue size.
- */
-static unsigned long long max_msgs_queue;
-
-/**
- * How often to send path keepalives. Paths timeout after 4 missed.
- */
-static struct GNUNET_TIME_Relative refresh_connection_time;
-
-/**
- * How often to send path create / ACKs.
- */
-static struct GNUNET_TIME_Relative create_connection_time;
-
-
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
-
-
-#if 0 // avoid compiler warning for unused static function
-static void
-fc_debug (struct CadetFlowControl *fc)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    IN: %u/%u\n",
-       ntohl (fc->last_pid_recv.pid),
-       ntohl (fc->last_ack_sent.pid));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    OUT: %u/%u\n",
-              fc->last_pid_sent, fc->last_ack_recv);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    QUEUE: %u/%u\n",
-              fc->queue_n, fc->queue_max);
-}
-
-static void
-connection_debug (struct CadetConnection *c)
-{
-  if (NULL == c)
-  {
-    LOG (GNUNET_ERROR_TYPE_INFO, "DEBUG NULL CONNECTION\n");
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Connection %s:%X\n",
-              peer2s (c->t->peer), GCC_2s (c));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  state: %u, pending msgs: %u\n",
-              c->state, c->pending_messages);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  FWD FC\n");
-  fc_debug (&c->fwd_fc);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  BCK FC\n");
-  fc_debug (&c->bck_fc);
-}
-#endif
-
-
-/**
- * Schedule next keepalive task, taking in consideration
- * the connection state and number of retries.
- *
- * @param c Connection for which to schedule the next keepalive.
- * @param fwd Direction for the next keepalive.
- */
-static void
-schedule_next_keepalive (struct CadetConnection *c, int fwd);
-
-
-/**
- * Resets the connection timeout task, some other message has done the
- * task's job.
- * - For the first peer on the direction this means to send
- *   a keepalive or a path confirmation message (either create or ACK).
- * - For all other peers, this means to destroy the connection,
- *   due to lack of activity.
- * Starts the timeout if no timeout was running (connection just created).
- *
- * @param c Connection whose timeout to reset.
- * @param fwd Is this forward?
- */
-static void
-connection_reset_timeout (struct CadetConnection *c, int fwd);
-
-
-/**
- * Get string description for tunnel state. Reentrant.
- *
- * @param s Tunnel state.
- *
- * @return String representation.
- */
-static const char *
-GCC_state2s (enum CadetConnectionState s)
-{
-  switch (s)
-  {
-    case CADET_CONNECTION_NEW:
-      return "CADET_CONNECTION_NEW";
-    case CADET_CONNECTION_SENT:
-      return "CADET_CONNECTION_SENT";
-    case CADET_CONNECTION_ACK:
-      return "CADET_CONNECTION_ACK";
-    case CADET_CONNECTION_READY:
-      return "CADET_CONNECTION_READY";
-    case CADET_CONNECTION_DESTROYED:
-      return "CADET_CONNECTION_DESTROYED";
-    case CADET_CONNECTION_BROKEN:
-      return "CADET_CONNECTION_BROKEN";
-    default:
-      GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_ERROR, " conn state %u unknown!\n", s);
-      return "CADET_CONNECTION_STATE_ERROR";
-  }
-}
-
-
 /**
- * Initialize a Flow Control structure to the initial state.
+ * Lookup a connection by its identifier.
  *
- * @param fc Flow Control structure to initialize.
+ * @param cid identifier to resolve
+ * @return NULL if connection was not found
  */
-static void
-fc_init (struct CadetFlowControl *fc)
-{
-  fc->next_pid.pid = 0;
-  fc->last_pid_sent.pid = htonl (UINT32_MAX);
-  fc->last_pid_recv.pid = htonl (UINT32_MAX);
-  fc->last_ack_sent.pid = (uint32_t) 0;
-  fc->last_ack_recv.pid = (uint32_t) 0;
-  fc->poll_task = NULL;
-  fc->poll_time = GNUNET_TIME_UNIT_SECONDS;
-  fc->queue_n = 0;
-  fc->queue_max = (max_msgs_queue / max_connections) + 1;
-}
-
-
-/**
- * Find a connection.
- *
- * @param cid Connection ID.
- *
- * @return conntection with the given ID @cid or NULL if not found.
- */
-static struct CadetConnection *
-connection_get (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
+struct CadetConnection *
+GCC_lookup (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
   return GNUNET_CONTAINER_multishortmap_get (connections,
                                              &cid->connection_of_tunnel);
@@ -482,3232 +181,911 @@ connection_get (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 
 
 /**
- * Change the connection state. Cannot change a connection marked as destroyed.
+ * Update the connection state. Also triggers the necessary
+ * MQM notifications.
  *
- * @param c Connection to change.
- * @param state New state to set.
+ * @param cc connection to update the state for
+ * @param new_state new state for @a cc
+ * @param new_mqm_ready new `mqm_ready` state for @a cc
  */
 static void
-connection_change_state (struct CadetConnection* c,
-                         enum CadetConnectionState state)
+update_state (struct CadetConnection *cc,
+              enum CadetConnectionState new_state,
+              int new_mqm_ready)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Connection %s state %s -> %s\n",
-       GCC_2s (c), GCC_state2s (c->state), GCC_state2s (state));
-  if (CADET_CONNECTION_DESTROYED <= c->state) /* Destroyed or broken. */
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "state not changing anymore\n");
-    return;
-  }
-  c->state = state;
-  if (CADET_CONNECTION_READY == state)
-    c->create_retry = 1;
-}
-
+  int old_ready;
+  int new_ready;
 
-/**
- * Mark a connection as "destroyed", to send all pending traffic and freeing
- * all associated resources, without accepting new status changes on it.
- *
- * @param c Connection to mark as destroyed.
- */
-static void
-mark_destroyed (struct CadetConnection *c)
-{
-  c->destroy = GNUNET_YES;
-  connection_change_state (c, CADET_CONNECTION_DESTROYED);
+  if ( (new_state == cc->state) &&
+       (new_mqm_ready == cc->mqm_ready) )
+    return; /* no change, nothing to do */
+  old_ready = ( (CADET_CONNECTION_READY == cc->state) &&
+                (GNUNET_YES == cc->mqm_ready) );
+  new_ready = ( (CADET_CONNECTION_READY == new_state) &&
+                (GNUNET_YES == new_mqm_ready) );
+  cc->state = new_state;
+  cc->mqm_ready = new_mqm_ready;
+  if (old_ready != new_ready)
+    cc->ready_cb (cc->ready_cb_cls,
+                  new_ready);
 }
 
 
 /**
- * Function called if a connection has been stalled for a while,
- * possibly due to a missed ACK. Poll the neighbor about its ACK status.
- *
- * @param cls Closure (poll ctx).
- */
-static void
-send_poll (void *cls);
-
-
-/**
- * Send an ACK on the connection, informing the predecessor about
- * the available buffer space. Should not be called in case the peer
- * is origin (no predecessor) in the @c fwd direction.
- *
- * Note that for fwd ack, the FWD mean forward *traffic* (root->dest),
- * the ACK itself goes "back" (dest->root).
+ * Destroy a connection, part of the internal implementation.  Called
+ * only from #GCC_destroy_from_core() or #GCC_destroy_from_tunnel().
  *
- * @param c Connection on which to send the ACK.
- * @param buffer How much space free to advertise?
- * @param fwd Is this FWD ACK? (Going dest -> root)
- * @param force Don't optimize out.
+ * @param cc connection to destroy
  */
 static void
-send_ack (struct CadetConnection *c,
-          unsigned int buffer,
-          int fwd,
-          int force)
+GCC_destroy (struct CadetConnection *cc)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetFlowControl *next_fc;
-  struct CadetFlowControl *prev_fc;
-  struct GNUNET_CADET_ConnectionEncryptedAckMessage msg;
-  struct CadetEncryptedMessageIdentifier ack_cemi;
-  int delta;
-
-  GCC_check_connections ();
-  GNUNET_assert (GNUNET_NO == GCC_is_origin (c, fwd));
-
-  next_fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  prev_fc = fwd ? &c->bck_fc : &c->fwd_fc;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "send %s ack on %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-
-  /* Check if we need to transmit the ACK. */
-  delta = ntohl (prev_fc->last_ack_sent.pid) - ntohl (prev_fc->last_pid_recv.pid);
-  if (3 < delta && buffer < delta && GNUNET_NO == force)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Not sending ACK, delta > 3\n");
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  last pid recv: %u, last ack sent: %u\n",
-         ntohl (prev_fc->last_pid_recv.pid),
-         ntohl (prev_fc->last_ack_sent.pid));
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Ok, ACK might be necessary, what PID to ACK? */
-  ack_cemi.pid = htonl (ntohl (prev_fc->last_pid_recv.pid) + buffer);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " ACK %u, last PID %u, last ACK %u, qmax %u, q %u\n",
-       ntohl (ack_cemi.pid),
-       ntohl (prev_fc->last_pid_recv.pid),
-       ntohl (prev_fc->last_ack_sent.pid),
-       next_fc->queue_max, next_fc->queue_n);
-  if ( (ack_cemi.pid == prev_fc->last_ack_sent.pid) &&
-       (GNUNET_NO == force) )
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Not sending FWD ACK, not needed\n");
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Check if message is already in queue */
-  if (NULL != prev_fc->ack_msg)
-  {
-    if (GC_is_pid_bigger (ntohl (ack_cemi.pid),
-                          ntohl (prev_fc->last_ack_sent.pid)))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " canceling old ACK\n");
-      GCC_cancel (prev_fc->ack_msg);
-      /* GCC_cancel triggers ack_sent(), which clears fc->ack_msg */
-    }
-    else
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " same ACK already in queue\n");
-      GCC_check_connections ();
-      return;
-    }
-  }
-  GNUNET_break (GC_is_pid_bigger (ntohl (ack_cemi.pid),
-                                  ntohl (prev_fc->last_ack_sent.pid)));
-  prev_fc->last_ack_sent = ack_cemi;
-
-  /* Build ACK message and send on conn */
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK);
-  msg.cemi_max = ack_cemi;
-  msg.cid = c->id;
-
-  prev_fc->ack_msg = GCC_send_prebuilt_message (&msg.header,
-                                                UINT16_MAX,
-                                                zero,
-                                                c,
-                                                !fwd,
-                                                GNUNET_YES,
-                                                NULL, NULL);
-  GNUNET_assert (NULL != prev_fc->ack_msg);
-  GCC_check_connections ();
-}
-
-
-/**
- * Update performance information if we are a connection's endpoint.
- *
- * @param c Connection to update.
- * @param wait How much time did we wait to send the last message.
- * @param size Size of the last message.
- */
-static void
-update_perf (struct CadetConnection *c,
-             struct GNUNET_TIME_Relative wait,
-             uint16_t size)
-{
-  struct CadetConnectionPerformance *p;
-  double usecsperbyte;
-
-  if (NULL == c->perf)
-    return; /* Only endpoints are interested in timing. */
-
-  p = c->perf;
-  usecsperbyte = ((double) wait.rel_value_us) / size;
-  if (p->size == AVG_MSGS)
-  {
-    /* Array is full. Substract oldest value, add new one and store. */
-    p->avg -= (p->usecsperbyte[p->idx] / AVG_MSGS);
-    p->usecsperbyte[p->idx] = usecsperbyte;
-    p->avg += (p->usecsperbyte[p->idx] / AVG_MSGS);
-  }
-  else
-  {
-    /* Array not yet full. Add current value to avg and store. */
-    p->usecsperbyte[p->idx] = usecsperbyte;
-    p->avg *= p->size;
-    p->avg += p->usecsperbyte[p->idx];
-    p->size++;
-    p->avg /= p->size;
-  }
-  p->idx = (p->idx + 1) % AVG_MSGS;
-}
-
-
-/**
- * Callback called when a connection queued message is sent.
- *
- * Calculates the average time and connection packet tracking.
- *
- * @param cls Closure (ConnectionQueue Handle), can be NULL.
- * @param c Connection this message was on.
- * @param fwd Was this a FWD going message?
- * @param sent Was it really sent? (Could have been canceled)
- * @param type Type of message sent.
- * @param payload_type Type of payload, if applicable.
- * @param pid Message ID, or 0 if not applicable (create, destroy, etc).
- * @param size Size of the message.
- * @param wait Time spent waiting for core (only the time for THIS message)
- */
-static void
-conn_message_sent (void *cls,
-                   struct CadetConnection *c,
-                   int fwd,
-                   int sent,
-                   uint16_t type,
-                   uint16_t payload_type,
-                   struct CadetEncryptedMessageIdentifier pid,
-                   size_t size,
-                   struct GNUNET_TIME_Relative wait)
-{
-  struct CadetConnectionQueue *q = cls;
-  struct CadetFlowControl *fc;
-  int forced;
-
-  GCC_check_connections ();
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         ">>> %s (%s %4u) on conn %s (%p) %s [%5u] in queue %s\n",
-         GC_m2s (type), GC_m2s (payload_type),
-         ntohl (pid.pid),
-         GCC_2s (c),
-         c,
-         GC_f2s (fwd), size,
-         GNUNET_STRINGS_relative_time_to_string (wait, GNUNET_YES));
-
-  /* If c is NULL, nothing to update. */
-  if (NULL == c)
-  {
-    if (type != GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN
-        && type != GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY)
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR, "Message %s sent on NULL connection!\n",
-           GC_m2s (type));
-    }
-    GCC_check_connections ();
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " %ssent %s %s pid %u\n",
-       sent ? "" : "not ", GC_f2s (fwd),
-       GC_m2s (type), GC_m2s (payload_type),
-       ntohl (pid.pid));
-  GCC_debug (c, GNUNET_ERROR_TYPE_DEBUG);
-
-  /* Update flow control info. */
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-
-  if (NULL != q)
+       "Destroying %s\n",
+       GCC_2s (cc));
+  if (NULL != cc->mq_man)
   {
-    GNUNET_CONTAINER_DLL_remove (fc->q_head, fc->q_tail, q);
-    forced = q->forced;
-    if (NULL != q->cont)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " calling cont\n");
-      q->cont (q->cont_cls, c, q, type, fwd, size);
-    }
-    GNUNET_free (q);
+    GCP_request_mq_cancel (cc->mq_man,
+                           NULL);
+    cc->mq_man = NULL;
   }
-  else /* CONN_CREATE or CONN_ACK */
+  if (NULL != cc->task)
   {
-    GNUNET_assert (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED != type);
-    forced = GNUNET_YES;
+    GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = NULL;
   }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " C_P- %p %u\n", c, c->pending_messages);
-  c->pending_messages--;
-  if ( (GNUNET_YES == c->destroy) &&
-       (0 == c->pending_messages) )
+  if (NULL != cc->keepalive_qe)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "!  destroying connection!\n");
-    GCC_destroy (c);
-    GCC_check_connections ();
-    return;
+    GCT_send_cancel (cc->keepalive_qe);
+    cc->keepalive_qe = NULL;
   }
-
-  /* Send ACK if needed, after accounting for sent ID in fc->queue_n */
-  switch (type)
-  {
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK:
-      c->maintenance_q = NULL;
-      /* Don't trigger a keepalive for sent ACKs, only SYN and SYNACKs */
-      if (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE == type || !fwd)
-        schedule_next_keepalive (c, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED:
-      if (GNUNET_YES == sent)
-      {
-        fc->last_pid_sent = pid;
-        if (GC_is_pid_bigger (ntohl (fc->last_pid_sent.pid) + 1,
-                              ntohl (fc->last_ack_recv.pid)) )
-          GCC_start_poll (c, fwd);
-        GCC_send_ack (c, fwd, GNUNET_NO);
-        connection_reset_timeout (c, fwd);
-      }
-
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "!  Q_N- %p %u\n", fc, fc->queue_n);
-      if (GNUNET_NO == forced)
-      {
-        fc->queue_n--;
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "!   accounting pid %u\n",
-             ntohl (fc->last_pid_sent.pid));
-      }
-      else
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "!   forced, Q_N not accounting pid %u\n",
-             ntohl (fc->last_pid_sent.pid));
-      }
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX:
-      if (GNUNET_YES == sent)
-        connection_reset_timeout (c, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL:
-      fc->poll_msg = NULL;
-      if (2 == c->destroy)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "POLL canceled on shutdown\n");
-        return;
-      }
-      if (0 == fc->queue_max)
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "POLL cancelled: neighbor disconnected\n");
-        return;
-      }
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "POLL sent for %s, scheduling new one!\n",
-          GCC_2s (c));
-      GNUNET_assert (NULL == fc->poll_task);
-      fc->poll_time = GNUNET_TIME_STD_BACKOFF (fc->poll_time);
-      fc->poll_task = GNUNET_SCHEDULER_add_delayed (fc->poll_time,
-                                                    &send_poll, fc);
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " task %u\n", fc->poll_task);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK:
-      fc->ack_msg = NULL;
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY:
-      break;
-
-    default:
-      LOG (GNUNET_ERROR_TYPE_ERROR, "%s unknown\n", GC_m2s (type));
-      GNUNET_break (0);
-      break;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "!  message sent!\n");
-
-  update_perf (c, wait, size);
-  GCC_check_connections ();
-}
-
-
-/**
- * Get the previous hop in a connection
- *
- * @param c Connection.
- *
- * @return Previous peer in the connection.
- */
-static struct CadetPeer *
-get_prev_hop (const struct CadetConnection *c)
-{
-  GNUNET_PEER_Id id;
-
-  if (NULL == c->path)
-    return NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " get prev hop %s [%u/%u]\n",
-       GCC_2s (c), c->own_pos, c->path->length);
-  if (0 == c->own_pos || c->path->length < 2)
-    id = c->path->peers[0];
-  else
-    id = c->path->peers[c->own_pos - 1];
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  ID: %s (%u)\n",
-       GNUNET_i2s (GNUNET_PEER_resolve2 (id)), id);
-
-  return GCP_get_short (id, GNUNET_YES);
-}
-
-
-/**
- * Get the next hop in a connection
- *
- * @param c Connection.
- *
- * @return Next peer in the connection.
- */
-static struct CadetPeer *
-get_next_hop (const struct CadetConnection *c)
-{
-  GNUNET_PEER_Id id;
-
-  if (NULL == c->path)
-    return NULL;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " get next hop %s [%u/%u]\n",
-       GCC_2s (c), c->own_pos, c->path->length);
-  if ((c->path->length - 1) == c->own_pos || c->path->length < 2)
-    id = c->path->peers[c->path->length - 1];
-  else
-    id = c->path->peers[c->own_pos + 1];
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  ID: %s (%u)\n",
-       GNUNET_i2s (GNUNET_PEER_resolve2 (id)), id);
-
-  return GCP_get_short (id, GNUNET_YES);
+  GCPP_del_connection (cc->path,
+                       cc->off,
+                       cc);
+  for (unsigned int i=0;i<cc->off;i++)
+    GCP_remove_connection (GCPP_get_peer_at_offset (cc->path,
+                                                    i),
+                           cc);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multishortmap_remove (connections,
+                                                        &GCC_get_id (cc)->connection_of_tunnel,
+                                                        cc));
+  GNUNET_free (cc);
 }
 
 
-/**
- * Check that the direct neighbours (previous and next hop)
- * are properly associated with this connection.
- *
- * @param c connection to check
- */
-static void
-check_neighbours (const struct CadetConnection *c)
-{
-  if (NULL == c->path)
-    return; /* nothing to check */
-  GCP_check_connection (get_next_hop (c), c);
-  GCP_check_connection (get_prev_hop (c), c);
-}
-
 
 /**
- * Helper for #GCC_check_connections().  Calls #check_neighbours().
+ * Destroy a connection, called when the CORE layer is already done
+ * (i.e. has received a BROKEN message), but if we still have to
+ * communicate the destruction of the connection to the tunnel (if one
+ * exists).
  *
- * @param cls NULL
- * @param key ignored
- * @param value the `struct CadetConnection` to check
- * @return #GNUNET_OK (continue to iterate)
- */
-static int
-check_connection (void *cls,
-                  const struct GNUNET_ShortHashCode *key,
-                  void *value)
-{
-  struct CadetConnection *c = value;
-
-  check_neighbours (c);
-  return GNUNET_OK;
-}
-
-
-/**
- * Check invariants for all connections using #check_neighbours().
+ * @param cc connection to destroy
  */
 void
-GCC_check_connections ()
-{
-  if (0 == CHECK_INVARIANTS)
-    return;
-  if (NULL == connections)
-    return;
-  GNUNET_CONTAINER_multishortmap_iterate (connections,
-                                          &check_connection,
-                                          NULL);
-}
-
-
-/**
- * Get the hop in a connection.
- *
- * @param c Connection.
- * @param fwd Next in the FWD direction?
- *
- * @return Next peer in the connection.
- */
-static struct CadetPeer *
-get_hop (struct CadetConnection *c, int fwd)
-{
-  return (fwd) ? get_next_hop (c) : get_prev_hop (c);
-}
-
-
-/**
- * Get a bit mask for a message received out-of-order.
- *
- * @param last_pid_recv Last PID we received prior to the out-of-order.
- * @param ooo_pid PID of the out-of-order message.
- */
-static uint32_t
-get_recv_bitmask (struct CadetEncryptedMessageIdentifier last_pid_recv,
-                  struct CadetEncryptedMessageIdentifier ooo_pid)
-{
-  // FIXME: should assert that the delta is in range...
-  return 1 << (ntohl (last_pid_recv.pid) - ntohl (ooo_pid.pid));
-}
-
-
-/**
- * Check is an out-of-order message is ok:
- * - at most 31 messages behind.
- * - not duplicate.
- *
- * @param last_pid_recv Last in-order PID received.
- */
-static int
-is_ooo_ok (struct CadetEncryptedMessageIdentifier last_pid_recv,
-           struct CadetEncryptedMessageIdentifier ooo_pid,
-           uint32_t ooo_bitmap)
-{
-  uint32_t mask;
-
-  if (GC_is_pid_bigger (ntohl (last_pid_recv.pid) - 31,
-                        ntohl (ooo_pid.pid)))
-    return GNUNET_NO;
-
-  mask = get_recv_bitmask (last_pid_recv,
-                           ooo_pid);
-  if (0 != (ooo_bitmap & mask))
-    return GNUNET_NO;
-
-  return GNUNET_YES;
-}
-
-
-/**
- * Is traffic coming from this sender 'FWD' traffic?
- *
- * @param c Connection to check.
- * @param sender Short peer identity of neighbor.
- *
- * @return #GNUNET_YES in case the sender is the 'prev' hop and therefore
- *         the traffic is 'FWD'.
- *         #GNUNET_NO for BCK.
- *         #GNUNET_SYSERR for errors (sender isn't a hop in the connection).
- */
-static int
-is_fwd (const struct CadetConnection *c,
-        const struct CadetPeer *sender)
-{
-  GNUNET_PEER_Id id;
-
-  id = GCP_get_short_id (sender);
-  if (GCP_get_short_id (get_prev_hop (c)) == id)
-    return GNUNET_YES;
-
-  if (GCP_get_short_id (get_next_hop (c)) == id)
-    return GNUNET_NO;
-
-  return GNUNET_SYSERR;
-}
-
-
-/**
- * Sends a CONNECTION ACK message in reponse to a received CONNECTION_CREATE
- * or a first CONNECTION_ACK directed to us.
- *
- * @param c Connection to confirm.
- * @param fwd Should we send it FWD? (root->dest)
- *            (First (~SYNACK) goes BCK, second (~ACK) goes FWD)
- */
-static void
-send_connection_ack (struct CadetConnection *c, int fwd)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct GNUNET_CADET_ConnectionCreateAckMessage msg;
-  struct CadetTunnel *t;
-  const uint16_t size = sizeof (struct GNUNET_CADET_ConnectionCreateAckMessage);
-  const uint16_t type = GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK;
-
-  GCC_check_connections ();
-  t = c->t;
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "==> %s ({ C %s ACK}    0) on conn %s (%p) %s [%5u]\n",
-       GC_m2s (type), GC_f2s (!fwd), GCC_2s (c), c, GC_f2s (fwd), size);
-
-  msg.header.size = htons (size);
-  msg.header.type = htons (type);
-  msg.reserved = htonl (0);
-  msg.cid = c->id;
-
-  GNUNET_assert (NULL == c->maintenance_q);
-  c->maintenance_q = GCP_send (get_hop (c, fwd),
-                               &msg.header,
-                               GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK,
-                               zero,
-                               c,
-                               fwd,
-                               &conn_message_sent, NULL);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  C_P+ %p %u (conn`ACK)\n",
-       c, c->pending_messages);
-  c->pending_messages++;
-
-  if (CADET_TUNNEL_NEW == GCT_get_cstate (t))
-    GCT_change_cstate (t, CADET_TUNNEL_WAITING);
-  if (CADET_CONNECTION_READY != c->state)
-    connection_change_state (c, CADET_CONNECTION_SENT);
-  GCC_check_connections ();
-}
-
-
-/**
- * Send a notification that a connection is broken.
- *
- * @param c Connection that is broken.
- * @param id1 Peer that has disconnected.
- * @param id2 Peer that has disconnected.
- * @param fwd Direction towards which to send it.
- */
-static void
-send_broken (struct CadetConnection *c,
-             const struct GNUNET_PeerIdentity *id1,
-             const struct GNUNET_PeerIdentity *id2,
-             int fwd)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct GNUNET_CADET_ConnectionBrokenMessage msg;
-
-  GCC_check_connections ();
-  msg.header.size = htons (sizeof (struct GNUNET_CADET_ConnectionBrokenMessage));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN);
-  msg.cid = c->id;
-  msg.reserved = htonl (0);
-  msg.peer1 = *id1;
-  msg.peer2 = *id2;
-  (void) GCC_send_prebuilt_message (&msg.header,
-                                    UINT16_MAX,
-                                    zero,
-                                    c,
-                                    fwd,
-                                    GNUNET_YES,
-                                    NULL, NULL);
-  GCC_check_connections ();
-}
-
-
-/**
- * Send a notification that a connection is broken, when a connection
- * isn't even known to the local peer or soon to be destroyed.
- *
- * @param connection_id Connection ID.
- * @param id1 Peer that has disconnected, probably local peer.
- * @param id2 Peer that has disconnected can be NULL if unknown.
- * @param neighbor Peer to notify (neighbor who sent the connection).
- */
-static void
-send_broken_unknown (const struct GNUNET_CADET_ConnectionTunnelIdentifier *connection_id,
-                     const struct GNUNET_PeerIdentity *id1,
-                     const struct GNUNET_PeerIdentity *id2,
-                     struct CadetPeer *neighbor)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct GNUNET_CADET_ConnectionBrokenMessage msg;
-
-  GCC_check_connections ();
-  LOG (GNUNET_ERROR_TYPE_INFO, "--> BROKEN on unknown connection %s\n",
-       GNUNET_sh2s (&connection_id->connection_of_tunnel));
-
-  msg.header.size = htons (sizeof (struct GNUNET_CADET_ConnectionBrokenMessage));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN);
-  msg.cid = *connection_id;
-  msg.reserved = htonl (0);
-  msg.peer1 = *id1;
-  if (NULL != id2)
-    msg.peer2 = *id2;
-  else
-    memset (&msg.peer2, 0, sizeof (msg.peer2));
-  GNUNET_assert (NULL != GCP_send (neighbor,
-                                   &msg.header,
-                                   UINT16_MAX,
-                                   zero,
-                                   NULL,
-                                   GNUNET_SYSERR, /* connection, fwd */
-                                   NULL, NULL)); /* continuation */
-  GCC_check_connections ();
-}
-
-
-/**
- * Send keepalive packets for a connection.
- *
- * @param c Connection to keep alive..
- * @param fwd Is this a FWD keepalive? (owner -> dest).
- */
-static void
-send_connection_keepalive (struct CadetConnection *c, int fwd)
+GCC_destroy_without_core (struct CadetConnection *cc)
 {
-  struct GNUNET_MessageHeader msg;
-  struct CadetFlowControl *fc;
-  int tunnel_ready;
-
-  GCC_check_connections ();
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "keepalive %s for connection %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-
-  GNUNET_assert (NULL != c->t);
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  tunnel_ready = GNUNET_YES == GCT_has_queued_traffic (c->t)
-                 && CADET_TUNNEL_KEY_OK <= GCT_get_estate (c->t);
-  if (0 < fc->queue_n || tunnel_ready)
+  if (NULL != cc->ct)
   {
-    LOG (GNUNET_ERROR_TYPE_INFO, "not sending keepalive, traffic in queue\n");
-    return;
+    GCT_connection_lost (cc->ct);
+    cc->ct = NULL;
   }
-
-  GNUNET_STATISTICS_update (stats, "# keepalives sent", 1, GNUNET_NO);
-
-  GNUNET_assert (NULL != c->t);
-  msg.size = htons (sizeof (msg));
-  msg.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE);
-
-  GNUNET_assert (NULL ==
-                 GCT_send_prebuilt_message (&msg, c->t, c,
-                                            GNUNET_NO, NULL, NULL));
-  GCC_check_connections ();
-}
-
-
-/**
- * Send CONNECTION_{CREATE/ACK} packets for a connection.
- *
- * @param c Connection for which to send the message.
- * @param fwd If #GNUNET_YES, send CREATE, otherwise send ACK.
- */
-static void
-connection_recreate (struct CadetConnection *c, int fwd)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "sending connection recreate\n");
-  if (fwd)
-    GCC_send_create (c);
-  else
-    send_connection_ack (c, GNUNET_NO);
+  GCC_destroy (cc);
 }
 
 
 /**
- * Generic connection timer management.
- * Depending on the role of the peer in the connection will send the
- * appropriate message (build or keepalive)
+ * Destroy a connection, called if the tunnel association with the
+ * connection was already broken, but we still need to notify the CORE
+ * layer about the breakage.
  *
- * @param c Conncetion to maintain.
- * @param fwd Is FWD?
+ * @param cc connection to destroy
  */
-static void
-connection_maintain (struct CadetConnection *c, int fwd)
+void
+GCC_destroy_without_tunnel (struct CadetConnection *cc)
 {
-  if (GNUNET_NO != c->destroy)
+  cc->ct = NULL;
+  if ( (CADET_CONNECTION_SENDING_CREATE != cc->state) &&
+       (NULL != cc->mq_man) )
   {
-    LOG (GNUNET_ERROR_TYPE_INFO, "not sending keepalive, being destroyed\n");
-    return;
-  }
-
-  if (NULL == c->t)
-  {
-    GNUNET_break (0);
-    GCC_debug (c, GNUNET_ERROR_TYPE_ERROR);
-    return;
-  }
+    struct GNUNET_MQ_Envelope *env;
+    struct GNUNET_CADET_ConnectionDestroyMessage *destroy_msg;
 
-  if (CADET_TUNNEL_SEARCHING == GCT_get_cstate (c->t))
-  {
-    /* If status is SEARCHING, why is there a connection? Should be WAITING */
-    GNUNET_break (0);
-    GCT_debug (c->t, GNUNET_ERROR_TYPE_ERROR);
-    LOG (GNUNET_ERROR_TYPE_INFO, "not sending keepalive, tunnel SEARCHING\n");
-    schedule_next_keepalive (c, fwd);
-    return;
-  }
-  switch (c->state)
-  {
-    case CADET_CONNECTION_NEW:
-      GNUNET_break (0);
-      /* fall-through */
-    case CADET_CONNECTION_SENT:
-      connection_recreate (c, fwd);
-      break;
-    case CADET_CONNECTION_READY:
-      send_connection_keepalive (c, fwd);
-      break;
-    default:
-      break;
+    /* Need to notify next hop that we are down. */
+    env = GNUNET_MQ_msg (destroy_msg,
+                         GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY);
+    destroy_msg->cid = cc->cid;
+    GCP_request_mq_cancel (cc->mq_man,
+                           env);
+    cc->mq_man = NULL;
   }
+  GCC_destroy (cc);
 }
 
 
 /**
- * Keep the connection alive.
+ * Return the tunnel associated with this connection.
  *
- * @param c Connection to keep alive.
- * @param fwd Direction.
+ * @param cc connection to query
+ * @return corresponding entry in the tunnel's connection list
  */
-static void
-connection_keepalive (struct CadetConnection *c,
-                      int fwd)
+struct CadetTConnection *
+GCC_get_ct (struct CadetConnection *cc)
 {
-  GCC_check_connections ();
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "%s keepalive for %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-
-  if (fwd)
-    c->fwd_maintenance_task = NULL;
-  else
-    c->bck_maintenance_task = NULL;
-  connection_maintain (c, fwd);
-  GCC_check_connections ();
-  /* Next execution will be scheduled by message_sent or _maintain*/
+  return cc->ct;
 }
 
 
 /**
- * Keep the connection alive in the FWD direction.
+ * Obtain performance @a metrics from @a cc.
  *
- * @param cls Closure (connection to keepalive).
+ * @param cc connection to query
+ * @return the metrics
  */
-static void
-connection_fwd_keepalive (void *cls)
+const struct CadetConnectionMetrics *
+GCC_get_metrics (struct CadetConnection *cc)
 {
-  struct CadetConnection *c = cls;
-
-  GCC_check_connections ();
-  connection_keepalive (c,
-                        GNUNET_YES);
-  GCC_check_connections ();
+  return &cc->metrics;
 }
 
 
 /**
- * Keep the connection alive in the BCK direction.
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE through the
+ * tunnel to prevent it from timing out.
  *
- * @param cls Closure (connection to keepalive).
+ * @param cls the `struct CadetConnection` to keep alive.
  */
 static void
-connection_bck_keepalive (void *cls)
-{
-  struct CadetConnection *c = cls;
-
-  GCC_check_connections ();
-  connection_keepalive (c,
-                        GNUNET_NO);
-  GCC_check_connections ();
-}
+send_keepalive (void *cls);
 
 
 /**
- * Schedule next keepalive task, taking in consideration
- * the connection state and number of retries.
+ * Keepalive was transmitted.  Remember this, and possibly
+ * schedule the next one.
  *
- * If the peer is not the origin, do nothing.
- *
- * @param c Connection for which to schedule the next keepalive.
- * @param fwd Direction for the next keepalive.
+ * @param cls the `struct CadetConnection` to keep alive.
+ * @param cid identifier of the connection within the tunnel, NULL
+ *            if transmission failed
  */
 static void
-schedule_next_keepalive (struct CadetConnection *c, int fwd)
+keepalive_done (void *cls,
+                const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  struct GNUNET_TIME_Relative delay;
-  struct GNUNET_SCHEDULER_Task * *task_id;
-  GNUNET_SCHEDULER_TaskCallback keepalive_task;
-
-  GCC_check_connections ();
-  if (GNUNET_NO == GCC_is_origin (c, fwd))
-    return;
-
-  /* Calculate delay to use, depending on the state of the connection */
-  if (CADET_CONNECTION_READY == c->state)
-  {
-    delay = refresh_connection_time;
-  }
-  else
-  {
-    if (1 > c->create_retry)
-      c->create_retry = 1;
-    delay = GNUNET_TIME_relative_saturating_multiply (create_connection_time,
-                                                      c->create_retry);
-    if (c->create_retry < 64) // TODO make configurable
-      c->create_retry *= 2;
-  }
-
-  /* Select direction-dependent parameters */
-  if (GNUNET_YES == fwd)
-  {
-    task_id = &c->fwd_maintenance_task;
-    keepalive_task = &connection_fwd_keepalive;
-  }
-  else
-  {
-    task_id = &c->bck_maintenance_task;
-    keepalive_task = &connection_bck_keepalive;
-  }
+  struct CadetConnection *cc = cls;
 
-  /* Check that no one scheduled it before us */
-  if (NULL != *task_id)
-  {
-    /* No need for a _break. It can happen for instance when sending a SYNACK
-     * for a duplicate SYN: the first SYNACK scheduled the task. */
-    GNUNET_SCHEDULER_cancel (*task_id);
-  }
-
-  /* Schedule the task */
-  *task_id = GNUNET_SCHEDULER_add_delayed (delay,
-                                           keepalive_task,
-                                           c);
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "next keepalive for %s in in %s\n",
-       GCC_2s (c), GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_YES));
-  GCC_check_connections ();
+  cc->keepalive_qe = NULL;
+  if ( (GNUNET_YES == cc->mqm_ready) &&
+       (NULL == cc->task) )
+    cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                             &send_keepalive,
+                                             cc);
 }
 
 
 /**
- * Cancel all transmissions that belong to a certain connection.
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE through the
+ * tunnel to prevent it from timing out.
  *
- * If the connection is scheduled for destruction and no more messages are left,
- * the connection will be destroyed by the continuation call.
- *
- * @param c Connection which to cancel. Might be destroyed during this call.
- * @param fwd Cancel fwd traffic?
+ * @param cls the `struct CadetConnection` to keep alive.
  */
 static void
-connection_cancel_queues (struct CadetConnection *c,
-                          int fwd)
+send_keepalive (void *cls)
 {
-  struct CadetFlowControl *fc;
+  struct CadetConnection *cc = cls;
+  struct GNUNET_MessageHeader msg;
 
-  GCC_check_connections ();
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Cancel %s queues for connection %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-  if (NULL == c)
+  cc->task = NULL;
+  if (CADET_TUNNEL_KEY_OK != GCT_get_estate (cc->ct->t))
   {
-    GNUNET_break (0);
+    /* Tunnel not yet ready, wait with keepalives... */
+    cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                             &send_keepalive,
+                                             cc);
     return;
   }
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  if (NULL != fc->poll_task)
-  {
-    GNUNET_SCHEDULER_cancel (fc->poll_task);
-    fc->poll_task = NULL;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  cancelled POLL task for fc %p\n", fc);
-  }
-  if (NULL != fc->poll_msg)
-  {
-    GCC_cancel (fc->poll_msg);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  cancelled POLL msg for fc %p\n", fc);
-  }
-
-  while (NULL != fc->q_head)
-  {
-    GCC_cancel (fc->q_head);
-  }
-  GCC_check_connections ();
-}
-
-
-/**
- * Function called if a connection has been stalled for a while,
- * possibly due to a missed ACK. Poll the neighbor about its ACK status.
- *
- * @param cls Closure (poll ctx).
- */
-static void
-send_poll (void *cls)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetFlowControl *fc = cls;
-  struct GNUNET_CADET_ConnectionHopByHopPollMessage msg;
-  struct CadetConnection *c;
-  int fwd;
-
-  fc->poll_task = NULL;
-  GCC_check_connections ();
-  c = fc->c;
-  fwd = fc == &c->fwd_fc;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Polling connection %s %s\n",
-       GCC_2s (c),  GC_f2s (fwd));
-
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL);
-  msg.header.size = htons (sizeof (msg));
-  msg.cid = c->id;
-  msg.cemi = fc->last_pid_sent;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " last pid sent: %u\n", ntohl (fc->last_pid_sent.pid));
-  fc->poll_msg
-    = GCC_send_prebuilt_message (&msg.header,
-                                 UINT16_MAX,
-                                 zero,
-                                 c,
-                                 fc == &c->fwd_fc,
-                                 GNUNET_YES,
-                                 NULL,
-                                 NULL);
-  GNUNET_assert (NULL != fc->poll_msg);
-  GCC_check_connections ();
-}
-
-
-/**
- * Generic connection timeout implementation.
- *
- * Timeout function due to lack of keepalive/traffic from an endpoint.
- * Destroys connection if called.
- *
- * @param c Connection to destroy.
- * @param fwd Was the timeout from the origin? (FWD timeout)
- */
-static void
-connection_timeout (struct CadetConnection *c, int fwd)
-{
-  GCC_check_connections ();
-
+  GNUNET_assert (NULL != cc->ct);
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  GNUNET_assert (NULL == cc->keepalive_qe);
   LOG (GNUNET_ERROR_TYPE_INFO,
-       "Connection %s %s timed out. Destroying.\n",
-       GCC_2s (c),
-       GC_f2s (fwd));
-  GCC_debug (c, GNUNET_ERROR_TYPE_DEBUG);
-
-  if (GCC_is_origin (c, fwd)) /* Loopback? Something is wrong! */
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* If dest, send "broken" notification. */
-  if (GCC_is_terminal (c, fwd))
-  {
-    struct CadetPeer *next_hop;
-
-    next_hop = fwd ? get_prev_hop (c) : get_next_hop (c);
-    send_broken_unknown (&c->id, &my_full_id, NULL, next_hop);
-  }
-
-  GCC_destroy (c);
-  GCC_check_connections ();
-}
-
-
-/**
- * Timeout function due to lack of keepalive/traffic from the owner.
- * Destroys connection if called.
- *
- * @param cls Closure (connection to destroy).
- */
-static void
-connection_fwd_timeout (void *cls)
-{
-  struct CadetConnection *c = cls;
-
-  c->fwd_maintenance_task = NULL;
-  GCC_check_connections ();
-  connection_timeout (c, GNUNET_YES);
-  GCC_check_connections ();
-}
-
-
-/**
- * Timeout function due to lack of keepalive/traffic from the destination.
- * Destroys connection if called.
- *
- * @param cls Closure (connection to destroy).
- */
-static void
-connection_bck_timeout (void *cls)
-{
-  struct CadetConnection *c = cls;
+       "Sending KEEPALIVE on behalf of %s via %s\n",
+       GCC_2s (cc),
+       GCT_2s (cc->ct->t));
+  GNUNET_STATISTICS_update (stats,
+                            "# keepalives sent",
+                            1,
+                            GNUNET_NO);
+  msg.size = htons (sizeof (msg));
+  msg.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE);
 
-  c->bck_maintenance_task = NULL;
-  GCC_check_connections ();
-  connection_timeout (c, GNUNET_NO);
-  GCC_check_connections ();
+  cc->keepalive_qe
+    = GCT_send (cc->ct->t,
+                &msg,
+                &keepalive_done,
+                cc);
 }
 
 
 /**
- * Resets the connection timeout task, some other message has done the
- * task's job.
- * - For the first peer on the direction this means to send
- *   a keepalive or a path confirmation message (either create or ACK).
- * - For all other peers, this means to destroy the connection,
- *   due to lack of activity.
- * Starts the timeout if no timeout was running (connection just created).
- *
- * @param c Connection whose timeout to reset.
- * @param fwd Is this forward?
+ * We sent a message for which we expect to receive an ACK via
+ * the connection identified by @a cti.
  *
- * TODO use heap to improve efficiency of scheduler.
+ * @param cid connection identifier where we expect an ACK
  */
-static void
-connection_reset_timeout (struct CadetConnection *c, int fwd)
+void
+GCC_ack_expected (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Connection %s reset timeout\n", GC_f2s (fwd));
-  if (GCC_is_origin (c, fwd)) /* Startpoint */
-  {
-    schedule_next_keepalive (c, fwd);
-    if (NULL != c->maintenance_q)
-    {
-      GCP_send_cancel (c->maintenance_q);
-      c->maintenance_q = NULL; /* Is set to NULL by conn_message_sent anyway */
-    }
-  }
-  else /* Relay, endpoint. */
-  {
-    struct GNUNET_TIME_Relative delay;
-    struct GNUNET_SCHEDULER_Task * *ti;
-    GNUNET_SCHEDULER_TaskCallback f;
-
-    ti = fwd ? &c->fwd_maintenance_task : &c->bck_maintenance_task;
+  struct CadetConnection *cc;
 
-    if (NULL != *ti)
-      GNUNET_SCHEDULER_cancel (*ti);
-    delay = GNUNET_TIME_relative_saturating_multiply (refresh_connection_time, 4);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  timing out in %s\n",
-         GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_NO));
-    f = fwd ? &connection_fwd_timeout : &connection_bck_timeout;
-    *ti = GNUNET_SCHEDULER_add_delayed (delay, f, c);
-  }
+  cc = GCC_lookup (cid);
+  if (NULL == cc)
+    return; /* whopise, connection alredy down? */
+  cc->metrics.num_acked_transmissions++;
 }
 
 
 /**
- * Iterator to compare each connection's path with the path of a new connection.
+ * We observed an ACK for a message that was originally sent via
+ * the connection identified by @a cti.
  *
- * If the connection coincides, the c member of path is set to the connection
- * and the destroy flag of the connection is set.
- *
- * @param cls Closure (new path).
- * @param c Connection in the tunnel to check.
+ * @param cti connection identifier where we got an ACK for a message
+ *            that was originally sent via this connection (the ACK
+ *            may have gotten back to us via a different connection).
  */
-static void
-check_path (void *cls, struct CadetConnection *c)
+void
+GCC_ack_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid)
 {
-  struct CadetConnection *new_conn = cls;
-  struct CadetPeerPath *path = new_conn->path;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  checking %s (%p), length %u\n",
-       GCC_2s (c), c, c->path->length);
+  struct CadetConnection *cc;
 
-  if (c != new_conn
-      && GNUNET_NO == c->destroy
-      && CADET_CONNECTION_BROKEN != c->state
-      && CADET_CONNECTION_DESTROYED != c->state
-      && path_equivalent (path, c->path))
-  {
-    new_conn->destroy = GNUNET_YES; /* Do not mark_destroyed, */
-    new_conn->path->c = c;          /* this is only a flag for the Iterator. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  MATCH!\n");
-  }
+  cc = GCC_lookup (cid);
+  if (NULL == cc)
+    return; /* whopise, connection alredy down? */
+  cc->metrics.num_successes++;
 }
 
 
 /**
- * Finds out if this path is already being used by an existing connection.
- *
- * Checks the tunnel towards the destination to see if it contains
- * any connection with the same path.
+ * We observed some the given @a latency on the connection
+ * identified by @a cti.  (The same connection was taken
+ * in both directions.)
  *
- * If the existing connection is ready, it is kept.
- * Otherwise if the sender has a smaller ID that ours, we accept it (and
- * the peer will eventually reject our attempt).
- *
- * @param path Path to check.
- * @return #GNUNET_YES if the tunnel has a connection with the same path,
- *         #GNUNET_NO otherwise.
+ * @param cid connection identifier where we measured latency
+ * @param latency the observed latency
  */
-static int
-does_connection_exist (struct CadetConnection *conn)
+void
+GCC_latency_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                      struct GNUNET_TIME_Relative latency)
 {
-  struct CadetPeer *p;
-  struct CadetTunnel *t;
-  struct CadetConnection *c;
-
-  p = GCP_get_short (conn->path->peers[0], GNUNET_NO);
-  if (NULL == p)
-    return GNUNET_NO;
-  t = GCP_get_tunnel (p);
-  if (NULL == t)
-    return GNUNET_NO;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Checking for duplicates\n");
+  struct CadetConnection *cc;
+  double weight;
+  double result;
 
-  GCT_iterate_connections (t, &check_path, conn);
-
-  if (GNUNET_YES == conn->destroy)
-  {
-    c = conn->path->c;
-    conn->destroy = GNUNET_NO;
-    conn->path->c = conn;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " found duplicate of %s\n", GCC_2s (conn));
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " duplicate: %s\n", GCC_2s (c));
-    GCC_debug (c, GNUNET_ERROR_TYPE_DEBUG);
-    if (CADET_CONNECTION_READY == c->state)
-    {
-      /* The other peer confirmed a live connection with this path,
-       * why are they trying to duplicate it? */
-      GNUNET_STATISTICS_update (stats, "# duplicate connections", 1, GNUNET_NO);
-      return GNUNET_YES;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " duplicate not ready, connection unique\n");
-    return GNUNET_NO;
-  }
+  cc = GCC_lookup (cid);
+  if (NULL == cc)
+    return; /* whopise, connection alredy down? */
+  GNUNET_STATISTICS_update (stats,
+                            "# latencies observed",
+                            1,
+                            GNUNET_NO);
+  cc->latency_datapoints++;
+  if (cc->latency_datapoints >= 7)
+    weight = 7.0;
   else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " %s has no duplicates\n", GCC_2s (conn));
-    return GNUNET_NO;
-  }
+    weight = cc->latency_datapoints;
+  /* Compute weighted average, giving at MOST weight 7 to the
+     existing values, or less if that value is based on fewer than 7
+     measurements. */
+  result = (weight * cc->metrics.aged_latency.rel_value_us) + 1.0 * latency.rel_value_us;
+  result /= (weight + 1.0);
+  cc->metrics.aged_latency.rel_value_us = (uint64_t) result;
 }
 
 
 /**
- * @brief Check if the tunnel this connection belongs to has any other
- * connection with the same path, and destroy one if so.
+ * A #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK was received for this connection, implying
+ * that the end-to-end connection is up.  Process it.
  *
- * @param cls Closure (connection to check).
- */
-static void
-check_duplicates (void *cls)
-{
-  struct CadetConnection *c = cls;
-
-  c->check_duplicates_task = NULL;
-  if (GNUNET_YES == does_connection_exist (c))
-  {
-    GCT_debug (c->t, GNUNET_ERROR_TYPE_DEBUG);
-    send_broken (c, &my_full_id, &my_full_id, GCC_is_origin (c, GNUNET_YES));
-    GCC_destroy (c);
-  }
-}
-
-
-/**
- * Wait for enough time to let any dead connections time out and check for
- * any remaining duplicates.
- *
- * @param c Connection that is a potential duplicate.
- */
-static void
-schedule_check_duplicates (struct CadetConnection *c)
-{
-  struct GNUNET_TIME_Relative delay;
-
-  if (NULL != c->check_duplicates_task)
-    return;
-  delay = GNUNET_TIME_relative_saturating_multiply (refresh_connection_time, 5);
-  c->check_duplicates_task = GNUNET_SCHEDULER_add_delayed (delay,
-                                                           &check_duplicates,
-                                                           c);
-}
-
-
-/**
- * Add the connection to the list of both neighbors.
- *
- * @param c Connection.
- *
- * @return #GNUNET_OK if everything went fine
- *         #GNUNET_SYSERR if the was an error and @c c is malformed.
- */
-static int
-register_neighbors (struct CadetConnection *c)
-{
-  c->next_peer = get_next_hop (c);
-  c->prev_peer = get_prev_hop (c);
-  GNUNET_assert (c->next_peer != c->prev_peer);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "register neighbors for connection %s\n",
-       GCC_2s (c));
-  path_debug (c->path);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "own pos %u\n", c->own_pos);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "putting connection %s to next peer %p\n",
-       GCC_2s (c),
-       c->next_peer);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "next peer %p %s\n",
-       c->next_peer,
-       GCP_2s (c->next_peer));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "putting connection %s to prev peer %p\n",
-       GCC_2s (c),
-       c->prev_peer);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "prev peer %p %s\n",
-       c->prev_peer,
-       GCP_2s (c->prev_peer));
-
-  if ( (GNUNET_NO == GCP_is_neighbor (c->next_peer)) ||
-       (GNUNET_NO == GCP_is_neighbor (c->prev_peer)) )
-  {
-    if (GCC_is_origin (c, GNUNET_YES))
-      GNUNET_STATISTICS_update (stats, "# local bad paths", 1, GNUNET_NO);
-    GNUNET_STATISTICS_update (stats, "# bad paths", 1, GNUNET_NO);
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  register neighbors failed\n");
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  prev: %s, neighbor?: %d\n",
-         GCP_2s (c->prev_peer),
-         GCP_is_neighbor (c->prev_peer));
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  next: %s, neighbor?: %d\n",
-         GCP_2s (c->next_peer),
-         GCP_is_neighbor (c->next_peer));
-    return GNUNET_SYSERR;
-  }
-  GCP_add_connection (c->next_peer, c, GNUNET_NO);
-  GCP_add_connection (c->prev_peer, c, GNUNET_YES);
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Remove the connection from the list of both neighbors.
- *
- * @param c Connection.
- */
-static void
-unregister_neighbors (struct CadetConnection *c)
-{
-//  struct CadetPeer *peer; FIXME dont use next_peer, prev_peer
-  /* Either already unregistered or never got registered, it's ok either way. */
-  if (NULL == c->path)
-    return;
-  if (NULL != c->next_peer)
-  {
-    GCP_remove_connection (c->next_peer, c);
-    c->next_peer = NULL;
-  }
-  if (NULL != c->prev_peer)
-  {
-    GCP_remove_connection (c->prev_peer, c);
-    c->prev_peer = NULL;
-  }
-}
-
-
-/**
- * Invalidates all paths towards all peers that comprise the connection which
- * rely on the disconnected peer.
- *
- * ~O(n^3) (peers in connection * paths/peer * links/path)
- *
- * @param c Connection whose peers' paths to clean.
- * @param disconnected Peer that disconnected.
- */
-static void
-invalidate_paths (struct CadetConnection *c,
-                  struct CadetPeer *disconnected)
-{
-  struct CadetPeer *peer;
-  unsigned int i;
-
-  for (i = 0; i < c->path->length; i++)
-  {
-    peer = GCP_get_short (c->path->peers[i], GNUNET_NO);
-    if (NULL != peer)
-      GCP_notify_broken_link (peer, &my_full_id, GCP_get_id (disconnected));
-  }
-}
-
-
-/**
- * Bind the connection to the peer and the tunnel to that peer.
- *
- * If the peer has no tunnel, create one. Update tunnel and connection
- * data structres to reflect new status.
- *
- * @param c Connection.
- * @param peer Peer.
- */
-static void
-add_to_peer (struct CadetConnection *c,
-             struct CadetPeer *peer)
-{
-  GCP_add_tunnel (peer);
-  c->t = GCP_get_tunnel (peer);
-  GCT_add_connection (c->t, c);
-}
-
-
-/**
- * Log receipt of message on stderr (INFO level).
- *
- * @param message Message received.
- * @param peer    Peer who sent the message.
- * @param conn_id Connection ID of the message.
- */
-static void
-log_message (const struct GNUNET_MessageHeader *message,
-             const struct CadetPeer *peer,
-             const struct GNUNET_CADET_ConnectionTunnelIdentifier *conn_id)
-{
-  uint16_t size;
-  uint16_t type;
-  char *arrow;
-
-  size = ntohs (message->size);
-  type = ntohs (message->type);
-  switch (type)
-  {
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY:
-      arrow = "==";
-      break;
-    default:
-      arrow = "--";
-  }
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "<%s %s on conn %s from %s, %6u bytes\n",
-       arrow,
-       GC_m2s (type),
-       GNUNET_sh2s (&conn_id->connection_of_tunnel),
-       GCP_2s(peer),
-       (unsigned int) size);
-}
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Handler for connection creation.
- *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc the connection that got the ACK.
  */
 void
-GCC_handle_create (struct CadetPeer *peer,
-                   const struct GNUNET_CADET_ConnectionCreateMessage *msg)
+GCC_handle_connection_create_ack (struct CadetConnection *cc)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid;
-  struct GNUNET_PeerIdentity *id;
-  struct CadetPeerPath *path;
-  struct CadetPeer *dest_peer;
-  struct CadetPeer *orig_peer;
-  struct CadetConnection *c;
-  unsigned int own_pos;
-  uint16_t size;
-
-  GCC_check_connections ();
-  size = ntohs (msg->header.size);
-
-  /* Calculate hops */
-  size -= sizeof (struct GNUNET_CADET_ConnectionCreateMessage);
-  if (0 != size % sizeof (struct GNUNET_PeerIdentity))
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-  size /= sizeof (struct GNUNET_PeerIdentity);
-  if (1 > size)
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    path has %u hops.\n", size);
-
-  /* Get parameters */
-  cid = &msg->cid;
-  log_message (&msg->header, peer, cid);
-  id = (struct GNUNET_PeerIdentity *) &msg[1];
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    origin: %s\n", GNUNET_i2s (id));
-
-  /* Create connection */
-  c = connection_get (cid);
-  if (NULL == c)
-  {
-    path = path_build_from_peer_ids ((struct GNUNET_PeerIdentity *) &msg[1],
-                                     size, myid, &own_pos);
-    if (NULL == path)
-    {
-      /* Path was malformed, probably our own ID was not in it. */
-      GNUNET_STATISTICS_update (stats, "# malformed paths", 1, GNUNET_NO);
-      GNUNET_break_op (0);
-      return;
-    }
-    if (0 == own_pos)
-    {
-      /* We received this request from a neighbor, we cannot be origin */
-      GNUNET_STATISTICS_update (stats, "# fake paths", 1, GNUNET_NO);
-      GNUNET_break_op (0);
-      path_destroy (path);
-      return;
-    }
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Own position: %u\n", own_pos);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Creating connection\n");
-    c = GCC_new (cid, NULL, path, own_pos);
-    if (NULL == c)
-    {
-      if (path->length - 1 == own_pos)
-      {
-        /* If we are destination, why did the creation fail? */
-        GNUNET_break (0);
-        path_destroy (path);
-        GCC_check_connections ();
-        return;
-      }
-      send_broken_unknown (cid, &my_full_id,
-                           GNUNET_PEER_resolve2 (path->peers[own_pos + 1]),
-                           peer);
-      path_destroy (path);
-      GCC_check_connections ();
-      return;
-    }
-    GCP_add_path_to_all (path, GNUNET_NO);
-    connection_reset_timeout (c, GNUNET_YES);
-  }
-  else
-  {
-    path = path_duplicate (c->path);
-  }
-  if (CADET_CONNECTION_NEW == c->state)
-    connection_change_state (c, CADET_CONNECTION_SENT);
-
-  /* Remember peers */
-  dest_peer = GCP_get (&id[size - 1], GNUNET_YES);
-  orig_peer = GCP_get (&id[0], GNUNET_YES);
-
-  /* Is it a connection to us? */
-  if (c->own_pos == path->length - 1)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  It's for us!\n");
-    GCP_add_path_to_origin (orig_peer, path_duplicate (path), GNUNET_YES);
-
-    add_to_peer (c, orig_peer);
-    if (GNUNET_YES == does_connection_exist (c))
-    {
-      /* Peer created a connection equal to one we think exists
-       * and is fine.
-       * Solution: Keep both and postpone disambiguation. In the meantime
-       * the connection will time out or peer will inform us it is broken.
-       *
-       * Other options:
-       * - Use explicit duplicate.
-       * - Accept new conn and destroy the old. (interruption in higher level)
-       * - Keep the one with higher ID / created by peer with higher ID. */
-       schedule_check_duplicates (c);
-    }
-
-    if (CADET_TUNNEL_NEW == GCT_get_cstate (c->t))
-      GCT_change_cstate (c->t,  CADET_TUNNEL_WAITING);
-    if (NULL == c->maintenance_q)
-      send_connection_ack (c, GNUNET_NO);
-    if (CADET_CONNECTION_SENT == c->state)
-      connection_change_state (c, CADET_CONNECTION_ACK);
-  }
-  else
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received CADET_CONNECTION_CREATE_ACK for %s in state %d (%s)\n",
+       GCC_2s (cc),
+       cc->state,
+       (GNUNET_YES == cc->mqm_ready) ? "MQM ready" : "MQM busy");
+  if (CADET_CONNECTION_READY == cc->state)
+    return; /* Duplicate ACK, ignore */
+  if (NULL != cc->task)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  not for us, retransmitting...\n");
-    GCP_add_path (dest_peer, path_duplicate (path), GNUNET_NO);
-    GCP_add_path_to_origin (orig_peer, path_duplicate (path), GNUNET_NO);
-    (void) GCC_send_prebuilt_message (&msg->header,
-                                      0,
-                                      zero,
-                                      c,
-                                      GNUNET_YES, GNUNET_YES,
-                                      NULL, NULL);
+    GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = NULL;
   }
-  path_destroy (path);
-  GCC_check_connections ();
+  cc->metrics.age = GNUNET_TIME_absolute_get ();
+  update_state (cc,
+                CADET_CONNECTION_READY,
+                cc->mqm_ready);
+  if ( (NULL == cc->keepalive_qe) &&
+       (GNUNET_YES == cc->mqm_ready) &&
+       (NULL == cc->task) )
+    cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                             &send_keepalive,
+                                             cc);
 }
 
 
 /**
- * Handler for connection confirmations.
+ * Handle KX message.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
  */
 void
-GCC_handle_confirm (struct CadetPeer *peer,
-                    const struct GNUNET_CADET_ConnectionCreateAckMessage *msg)
+GCC_handle_kx (struct CadetConnection *cc,
+               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetConnection *c;
-  enum CadetConnectionState oldstate;
-  int fwd;
-
-  GCC_check_connections ();
-  log_message (&msg->header, peer, &msg->cid);
-  c = connection_get (&msg->cid);
-  if (NULL == c)
-  {
-    GNUNET_STATISTICS_update (stats, "# control on unknown connection",
-                              1, GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  don't know the connection!\n");
-    send_broken_unknown (&msg->cid, &my_full_id, NULL, peer);
-    GCC_check_connections ();
-    return;
-  }
-  if (GNUNET_NO != c->destroy)
+  if (CADET_CONNECTION_SENT == cc->state)
   {
-    GNUNET_assert (CADET_CONNECTION_DESTROYED == c->state);
-    GNUNET_STATISTICS_update (stats, "# control on dying connection",
-                              1, GNUNET_NO);
+    /* We didn't get the CADET_CONNECTION_CREATE_ACK, but instead got payload. That's fine,
+       clearly something is working, so pretend we got an ACK. */
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "connection %s being destroyed, ignoring confirm\n",
-         GCC_2s (c));
-    GCC_check_connections ();
-    return;
-  }
-
-  oldstate = c->state;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  via peer %s\n", GCP_2s (peer));
-  if (get_next_hop (c) == peer)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  SYNACK\n");
-    fwd = GNUNET_NO;
-    if (CADET_CONNECTION_SENT == oldstate)
-      connection_change_state (c, CADET_CONNECTION_ACK);
-  }
-  else if (get_prev_hop (c) == peer)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  FINAL ACK\n");
-    fwd = GNUNET_YES;
-    connection_change_state (c, CADET_CONNECTION_READY);
-  }
-  else
-  {
-    GNUNET_STATISTICS_update (stats, "# control on connection from wrong peer",
-                              1, GNUNET_NO);
-    GNUNET_break_op (0);
-    return;
-  }
-
-  connection_reset_timeout (c, fwd);
-
-  GNUNET_assert (NULL != c->path);
-  GCP_add_path_to_all (c->path, GNUNET_YES);
-
-  /* Message for us as creator? */
-  if (GNUNET_YES == GCC_is_origin (c, GNUNET_YES))
-  {
-    if (GNUNET_NO != fwd)
-    {
-      GNUNET_break (0);
-      return;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Connection (SYN)ACK for us!\n");
-
-    /* If just created, cancel the short timeout and start a long one */
-    if (CADET_CONNECTION_SENT == oldstate)
-    {
-      c->create_retry = 1;
-      connection_reset_timeout (c, GNUNET_YES);
-    }
-
-    /* Change connection state, send ACK */
-    connection_change_state (c, CADET_CONNECTION_READY);
-    send_connection_ack (c, GNUNET_YES);
-
-    /* Change tunnel state, trigger KX */
-    if (CADET_TUNNEL_WAITING == GCT_get_cstate (c->t))
-      GCT_change_cstate (c->t, CADET_TUNNEL_READY);
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Message for us as destination? */
-  if (GCC_is_terminal (c, GNUNET_YES))
-  {
-    if (GNUNET_YES != fwd)
-    {
-      GNUNET_break (0);
-      return;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Connection ACK for us!\n");
-
-    /* If just created, cancel the short timeout and start a long one */
-    if (CADET_CONNECTION_ACK == oldstate)
-      connection_reset_timeout (c, GNUNET_NO);
-
-    /* Change tunnel state */
-    if (CADET_TUNNEL_WAITING == GCT_get_cstate (c->t))
-      GCT_change_cstate (c->t, CADET_TUNNEL_READY);
-    GCC_check_connections ();
+         "Faking connection CADET_CONNECTION_CREATE_ACK for %s due to KX\n",
+         GCC_2s (cc));
+    GCC_handle_connection_create_ack (cc);
   }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  not for us, retransmitting...\n");
-    (void) GCC_send_prebuilt_message (&msg->header, 0,
-                                      zero,
-                                      c,
-                                      fwd,
-                                      GNUNET_YES, NULL, NULL);
-  }
-  GCC_check_connections ();
+  GCT_handle_kx (cc->ct,
+                 msg);
 }
 
 
 /**
- * Handler for notifications of broken connections.
+ * Handle KX_AUTH message.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
  */
 void
-GCC_handle_broken (struct CadetPeer *peer,
-                   const struct GNUNET_CADET_ConnectionBrokenMessage *msg)
+GCC_handle_kx_auth (struct CadetConnection *cc,
+                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetConnection *c;
-  struct CadetTunnel *t;
-  int fwd;
-
-  GCC_check_connections ();
-  log_message (&msg->header, peer, &msg->cid);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  regarding %s\n", GNUNET_i2s (&msg->peer1));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  regarding %s\n", GNUNET_i2s (&msg->peer2));
-  c = connection_get (&msg->cid);
-  if (NULL == c)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  duplicate CONNECTION_BROKEN\n");
-    GNUNET_STATISTICS_update (stats, "# duplicate CONNECTION_BROKEN",
-                              1, GNUNET_NO);
-    GCC_check_connections ();
-    return;
-  }
-
-  t = c->t;
-
-  fwd = is_fwd (c, peer);
-  if (GNUNET_SYSERR == fwd)
-  {
-    GNUNET_break_op (0);
-    GCC_check_connections ();
-    return;
-  }
-  mark_destroyed (c);
-  if (GCC_is_terminal (c, fwd))
-  {
-    struct CadetPeer *endpoint;
-
-    if (NULL == t)
-    {
-      /* A terminal connection should not have 't' set to NULL. */
-      GNUNET_break (0);
-      GCC_debug (c, GNUNET_ERROR_TYPE_ERROR);
-      return;
-    }
-    endpoint = GCP_get_short (c->path->peers[c->path->length - 1], GNUNET_YES);
-    if (2 < c->path->length)
-      path_invalidate (c->path);
-    GCP_notify_broken_link (endpoint, &msg->peer1, &msg->peer2);
-
-    connection_change_state (c, CADET_CONNECTION_BROKEN);
-    GCT_remove_connection (t, c);
-    c->t = NULL;
-
-    GCC_destroy (c);
-  }
-  else
+  if (CADET_CONNECTION_SENT == cc->state)
   {
-    (void) GCC_send_prebuilt_message (&msg->header, 0,
-                                      zero, c, fwd,
-                                      GNUNET_YES, NULL, NULL);
-    connection_cancel_queues (c, !fwd);
+    /* We didn't get the CADET_CONNECTION_CREATE_ACK, but instead got payload. That's fine,
+       clearly something is working, so pretend we got an ACK. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Faking connection CADET_CONNECTION_CREATE_ACK for %s due to KX\n",
+         GCC_2s (cc));
+    GCC_handle_connection_create_ack (cc);
   }
-  GCC_check_connections ();
-  return;
+  GCT_handle_kx_auth (cc->ct,
+                      msg);
 }
 
 
 /**
- * Handler for notifications of destroyed connections.
+ * Handle encrypted message.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection that received encrypted message
+ * @param msg the encrypted message to decrypt
  */
 void
-GCC_handle_destroy (struct CadetPeer *peer,
-                    const struct GNUNET_CADET_ConnectionDestroyMessage *msg)
+GCC_handle_encrypted (struct CadetConnection *cc,
+                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetConnection *c;
-  int fwd;
-
-  GCC_check_connections ();
-  log_message (&msg->header, peer, &msg->cid);
-  c = connection_get (&msg->cid);
-  if (NULL == c)
+  if (CADET_CONNECTION_SENT == cc->state)
   {
-    /* Probably already got the message from another path,
-     * destroyed the tunnel and retransmitted to children.
-     * Safe to ignore.
-     */
-    GNUNET_STATISTICS_update (stats,
-                              "# control on unknown connection",
-                              1, GNUNET_NO);
+    /* We didn't get the CREATE_ACK, but instead got payload. That's fine,
+       clearly something is working, so pretend we got an ACK. */
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  connection unknown destroyed: previously destroyed?\n");
-    GCC_check_connections ();
-    return;
-  }
-
-  fwd = is_fwd (c, peer);
-  if (GNUNET_SYSERR == fwd)
-  {
-    GNUNET_break_op (0);
-    GCC_check_connections ();
-    return;
-  }
-
-  if (GNUNET_NO == GCC_is_terminal (c, fwd))
-  {
-    (void) GCC_send_prebuilt_message (&msg->header, 0,
-                                      zero, c, fwd,
-                                      GNUNET_YES, NULL, NULL);
+         "Faking connection ACK for %s due to ENCRYPTED payload\n",
+         GCC_2s (cc));
+    GCC_handle_connection_create_ack (cc);
   }
-  else if (0 == c->pending_messages)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  directly destroying connection!\n");
-    GCC_destroy (c);
-    GCC_check_connections ();
-    return;
-  }
-  mark_destroyed (c);
-  if (NULL != c->t)
-  {
-    GCT_remove_connection (c->t, c);
-    c->t = NULL;
-  }
-  GCC_check_connections ();
-  return;
+  cc->metrics.last_use = GNUNET_TIME_absolute_get ();
+  GCT_handle_encrypted (cc->ct,
+                        msg);
 }
 
 
 /**
- * Handler for cadet network traffic hop-by-hop acks.
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE message to the
+ * first hop.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cls the `struct CadetConnection` to initiate
  */
-void
-GCC_handle_ack (struct CadetPeer *peer,
-                const struct GNUNET_CADET_ConnectionEncryptedAckMessage *msg)
-{
-  struct CadetConnection *c;
-  struct CadetFlowControl *fc;
-  struct CadetEncryptedMessageIdentifier ack;
-  int fwd;
-
-  GCC_check_connections ();
-  log_message (&msg->header, peer, &msg->cid);
-  c = connection_get (&msg->cid);
-  if (NULL == c)
-  {
-    GNUNET_STATISTICS_update (stats,
-                              "# ack on unknown connection",
-                              1,
-                              GNUNET_NO);
-    send_broken_unknown (&msg->cid,
-                         &my_full_id,
-                         NULL,
-                         peer);
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Is this a forward or backward ACK? */
-  if (get_next_hop (c) == peer)
-  {
-    fc = &c->fwd_fc;
-    fwd = GNUNET_YES;
-  }
-  else if (get_prev_hop (c) == peer)
-  {
-    fc = &c->bck_fc;
-    fwd = GNUNET_NO;
-  }
-  else
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-
-  ack = msg->cemi_max;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " %s ACK %u (was %u)\n",
-       GC_f2s (fwd),
-       ntohl (ack.pid),
-       ntohl (fc->last_ack_recv.pid));
-  if (GC_is_pid_bigger (ntohl (ack.pid),
-                        ntohl (fc->last_ack_recv.pid)))
-    fc->last_ack_recv = ack;
-
-  /* Cancel polling if the ACK is big enough. */
-  if ( (NULL != fc->poll_task) &
-       GC_is_pid_bigger (ntohl (fc->last_ack_recv.pid),
-                         ntohl (fc->last_pid_sent.pid)))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Cancel poll\n");
-    GNUNET_SCHEDULER_cancel (fc->poll_task);
-    fc->poll_task = NULL;
-    fc->poll_time = GNUNET_TIME_UNIT_SECONDS;
-  }
-
-  GCC_check_connections ();
+static void
+send_create (void *cls)
+{
+  struct CadetConnection *cc = cls;
+  struct GNUNET_CADET_ConnectionCreateMessage *create_msg;
+  struct GNUNET_PeerIdentity *pids;
+  struct GNUNET_MQ_Envelope *env;
+  unsigned int path_length;
+
+  cc->task = NULL;
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  path_length = GCPP_get_length (cc->path);
+  env = GNUNET_MQ_msg_extra (create_msg,
+                             (1 + path_length) * sizeof (struct GNUNET_PeerIdentity),
+                             GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE);
+  create_msg->options = htonl ((uint32_t) cc->options);
+  create_msg->cid = cc->cid;
+  pids = (struct GNUNET_PeerIdentity *) &create_msg[1];
+  pids[0] = my_full_id;
+  for (unsigned int i=0;i<path_length;i++)
+    pids[i + 1] = *GCP_get_id (GCPP_get_peer_at_offset (cc->path,
+                                                        i));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending CADET_CONNECTION_CREATE message for %s\n",
+       GCC_2s (cc));
+  cc->env = env;
+  update_state (cc,
+                CADET_CONNECTION_SENT,
+                GNUNET_NO);
+  GCP_send (cc->mq_man,
+            env);
 }
 
 
 /**
- * Handler for cadet network traffic hop-by-hop data counter polls.
+ * Send a CREATE_ACK message towards the origin.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cls the `struct CadetConnection` to initiate
  */
-void
-GCC_handle_poll (struct CadetPeer *peer,
-                 const struct GNUNET_CADET_ConnectionHopByHopPollMessage *msg)
+static void
+send_create_ack (void *cls)
 {
-  struct CadetConnection *c;
-  struct CadetFlowControl *fc;
-  struct CadetEncryptedMessageIdentifier pid;
-  int fwd;
-
-  GCC_check_connections ();
-  log_message (&msg->header, peer, &msg->cid);
-  c = connection_get (&msg->cid);
-  if (NULL == c)
-  {
-    GNUNET_STATISTICS_update (stats, "# poll on unknown connection", 1,
-                              GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "POLL message on unknown connection %s!\n",
-         GNUNET_sh2s (&msg->cid.connection_of_tunnel));
-    send_broken_unknown (&msg->cid,
-                         &my_full_id,
-                         NULL,
-                         peer);
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Is this a forward or backward ACK?
-   * Note: a poll should never be needed in a loopback case,
-   * since there is no possiblility of packet loss there, so
-   * this way of discerining FWD/BCK should not be a problem.
-   */
-  if (get_next_hop (c) == peer)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  FWD FC\n");
-    fc = &c->fwd_fc;
-  }
-  else if (get_prev_hop (c) == peer)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  BCK FC\n");
-    fc = &c->bck_fc;
-  }
-  else
-  {
-    GNUNET_break_op (0);
-    return;
-  }
+  struct CadetConnection *cc = cls;
+  struct GNUNET_CADET_ConnectionCreateAckMessage *ack_msg;
+  struct GNUNET_MQ_Envelope *env;
 
-  pid = msg->cemi;
+  cc->task = NULL;
+  GNUNET_assert (CADET_CONNECTION_CREATE_RECEIVED == cc->state);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "  PID %u, OLD %u\n",
-       ntohl (pid.pid),
-       ntohl (fc->last_pid_recv.pid));
-  fc->last_pid_recv = pid;
-  fwd = fc == &c->bck_fc;
-  GCC_send_ack (c, fwd, GNUNET_YES);
-  GCC_check_connections ();
+       "Sending CONNECTION_CREATE_ACK message for %s\n",
+       GCC_2s (cc));
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  env = GNUNET_MQ_msg (ack_msg,
+                       GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK);
+  ack_msg->cid = cc->cid;
+  cc->env = env;
+  update_state (cc,
+                CADET_CONNECTION_READY,
+                GNUNET_NO);
+  GCP_send (cc->mq_man,
+            env);
 }
 
 
 /**
- * Check the message against internal state and test if it goes FWD or BCK.
- *
- * Updates the PID, state and timeout values for the connection.
- *
- * @param message Message to check. It must belong to an existing connection.
- * @param cid Connection ID (even if @a c is NULL, the ID is still needed).
- * @param c Connection this message should belong. If NULL, check fails.
- * @param sender Neighbor that sent the message.
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE for a
+ * connection that we already have.  Either our ACK got lost
+ * or something is fishy.  Consider retransmitting the ACK.
  *
- * @return #GNUNET_YES if the message goes FWD.
- *         #GNUNET_NO if it goes BCK.
- *         #GNUNET_SYSERR if there is an error (unauthorized sender, ...).
+ * @param cc connection that got the duplicate CREATE
  */
-static int
-check_message (const struct GNUNET_MessageHeader *message,
-               const struct GNUNET_CADET_ConnectionTunnelIdentifier* cid,
-               struct CadetConnection *c,
-               struct CadetPeer *sender,
-               struct CadetEncryptedMessageIdentifier pid)
+void
+GCC_handle_duplicate_create (struct CadetConnection *cc)
 {
-  struct CadetFlowControl *fc;
-  struct CadetPeer *hop;
-  int fwd;
-  uint16_t type;
-
-  /* Check connection */
-  if (NULL == c)
+  if (GNUNET_YES == cc->mqm_ready)
   {
-    GNUNET_STATISTICS_update (stats,
-                              "# unknown connection",
-                              1, GNUNET_NO);
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "%s on unknown connection %s\n",
-         GC_m2s (ntohs (message->type)),
-         GNUNET_sh2s (&cid->connection_of_tunnel));
-    GNUNET_break_op (0);
-    send_broken_unknown (cid,
-                         &my_full_id,
-                         NULL,
-                         sender);
-    return GNUNET_SYSERR;
-  }
-
-  /* Check if origin is as expected */
-  hop = get_prev_hop (c);
-  if (sender == hop)
-  {
-    fwd = GNUNET_YES;
+         "Got duplicate CREATE for %s, scheduling another ACK (%s)\n",
+         GCC_2s (cc),
+         (GNUNET_YES == cc->mqm_ready) ? "MQM ready" : "MQM busy");
+    /* Revert back to the state of having only received the 'CREATE',
+       and immediately proceed to send the CREATE_ACK. */
+    update_state (cc,
+                  CADET_CONNECTION_CREATE_RECEIVED,
+                  cc->mqm_ready);
+    if (NULL != cc->task)
+      GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = GNUNET_SCHEDULER_add_now (&send_create_ack,
+                                         cc);
   }
   else
   {
-    hop = get_next_hop (c);
-    GNUNET_break (hop == c->next_peer);
-    if (sender == hop)
-    {
-      fwd = GNUNET_NO;
-    }
-    else
-    {
-      /* Unexpected peer sending traffic on a connection. */
-      GNUNET_break_op (0);
-      return GNUNET_SYSERR;
-    }
-  }
-
-  /* Check PID for payload messages */
-  type = ntohs (message->type);
-  if (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED == type)
-  {
-    fc = fwd ? &c->bck_fc : &c->fwd_fc;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " PID %u (expected in interval [%u,%u])\n",
-         ntohl (pid.pid),
-         ntohl (fc->last_pid_recv.pid) + 1,
-         ntohl (fc->last_ack_sent.pid));
-    if (GC_is_pid_bigger (ntohl (pid.pid),
-                          ntohl (fc->last_ack_sent.pid)))
-    {
-      GNUNET_STATISTICS_update (stats,
-                                "# unsolicited message",
-                                1,
-                                GNUNET_NO);
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           "Received PID %u, (prev %u), ACK %u\n",
-           pid, fc->last_pid_recv, fc->last_ack_sent);
-      return GNUNET_SYSERR;
-    }
-    if (GC_is_pid_bigger (ntohl (pid.pid),
-                          ntohl (fc->last_pid_recv.pid)))
-    {
-      unsigned int delta;
-
-      delta = ntohl (pid.pid) - ntohl (fc->last_pid_recv.pid);
-      fc->last_pid_recv = pid;
-      fc->recv_bitmap <<= delta;
-      fc->recv_bitmap |= 1;
-    }
-    else
-    {
-      GNUNET_STATISTICS_update (stats,
-                                "# out of order PID",
-                                1,
-                                GNUNET_NO);
-      if (GNUNET_NO == is_ooo_ok (fc->last_pid_recv,
-                                  pid,
-                                  fc->recv_bitmap))
-      {
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             "PID %u unexpected (%u+), dropping!\n",
-             ntohl (pid.pid),
-             ntohl (fc->last_pid_recv.pid) - 31);
-        return GNUNET_SYSERR;
-      }
-      fc->recv_bitmap |= get_recv_bitmask (fc->last_pid_recv,
-                                           pid);
-    }
-  }
-
-  /* Count as connection confirmation. */
-  if ( (CADET_CONNECTION_SENT == c->state) ||
-       (CADET_CONNECTION_ACK == c->state) )
-  {
-    connection_change_state (c, CADET_CONNECTION_READY);
-    if (NULL != c->t)
-    {
-      if (CADET_TUNNEL_WAITING == GCT_get_cstate (c->t))
-        GCT_change_cstate (c->t, CADET_TUNNEL_READY);
-    }
+    /* We are currently sending something else back, which
+       can only be an ACK or payload, either of which would
+       do. So actually no need to do anything. */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got duplicate CREATE for %s. MQ is busy, not queueing another ACK\n",
+         GCC_2s (cc));
   }
-  connection_reset_timeout (c, fwd);
-
-  return fwd;
 }
 
 
 /**
- * Handler for key exchange traffic (Axolotl KX).
+ * There has been a change in the message queue existence for our
+ * peer at the first hop.  Adjust accordingly.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cls the `struct CadetConnection`
+ * @param available #GNUNET_YES if sending is now possible,
+ *                  #GNUNET_NO if sending is no longer possible
+ *                  #GNUNET_SYSERR if sending is no longer possible
+ *                                 and the last envelope was discarded
  */
-void
-GCC_handle_kx (struct CadetPeer *peer,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
+static void
+manage_first_hop_mq (void *cls,
+                     int available)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  const struct GNUNET_CADET_ConnectionTunnelIdentifier* cid;
-  struct CadetConnection *c;
-  int fwd;
-
-  GCC_check_connections ();
-  cid = &msg->cid;
-  log_message (&msg->header, peer, cid);
-
-  c = connection_get (cid);
-  fwd = check_message (&msg->header,
-                       cid,
-                       c,
-                       peer,
-                       zero);
-
-  /* If something went wrong, discard message. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    GNUNET_break_op (0);
-    GCC_check_connections ();
-    return;
-  }
+  struct CadetConnection *cc = cls;
 
-  /* Is this message for us? */
-  if (GCC_is_terminal (c, fwd))
+  if (GNUNET_YES != available)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  message for us!\n");
-    GNUNET_STATISTICS_update (stats, "# received KX", 1, GNUNET_NO);
-    if (NULL == c->t)
+    /* Connection is down, for now... */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Core MQ for %s went down\n",
+         GCC_2s (cc));
+    update_state (cc,
+                  CADET_CONNECTION_NEW,
+                  GNUNET_NO);
+    cc->retry_delay = GNUNET_TIME_UNIT_ZERO;
+    if (NULL != cc->task)
     {
-      GNUNET_break (0);
-      return;
+      GNUNET_SCHEDULER_cancel (cc->task);
+      cc->task = NULL;
     }
-    GCT_handle_kx (c->t, msg);
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Message not for us: forward to next hop */
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  not for us, retransmitting...\n");
-  GNUNET_STATISTICS_update (stats, "# messages forwarded", 1, GNUNET_NO);
-  (void) GCC_send_prebuilt_message (&msg->header, 0,
-                                    zero, c, fwd,
-                                    GNUNET_NO, NULL, NULL);
-  GCC_check_connections ();
-}
-
-
-/**
- * Handler for encrypted cadet network traffic (channel mgmt, data).
- *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
- */
-void
-GCC_handle_encrypted (struct CadetPeer *peer,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  const struct GNUNET_CADET_ConnectionTunnelIdentifier* cid;
-  struct CadetConnection *c;
-  struct CadetEncryptedMessageIdentifier pid;
-  int fwd;
-
-  GCC_check_connections ();
-  cid = &msg->cid;
-  pid = msg->cemi;
-  log_message (&msg->header, peer, cid);
-
-  c = connection_get (cid);
-  fwd = check_message (&msg->header,
-                       cid,
-                       c,
-                       peer,
-                       pid);
-
-  /* If something went wrong, discard message. */
-  if (GNUNET_SYSERR == fwd)
-  {
-    GCC_check_connections ();
     return;
   }
 
-  /* Is this message for us? */
-  if (GCC_is_terminal (c, fwd))
-  {
-    GNUNET_STATISTICS_update (stats, "# received encrypted", 1, GNUNET_NO);
-
-    if (NULL == c->t)
+  update_state (cc,
+                cc->state,
+                GNUNET_YES);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Core MQ for %s became available in state %d\n",
+       GCC_2s (cc),
+       cc->state);
+  switch (cc->state)
+  {
+  case CADET_CONNECTION_NEW:
+    /* Transmit immediately */
+    cc->task = GNUNET_SCHEDULER_add_now (&send_create,
+                                         cc);
+    break;
+  case CADET_CONNECTION_SENDING_CREATE:
+    /* Should not be possible to be called in this state. */
+    GNUNET_assert (0);
+    break;
+  case CADET_CONNECTION_SENT:
+    /* Retry a bit later... */
+    cc->retry_delay = GNUNET_TIME_STD_BACKOFF (cc->retry_delay);
+    cc->task = GNUNET_SCHEDULER_add_delayed (cc->retry_delay,
+                                             &send_create,
+                                             cc);
+    break;
+  case CADET_CONNECTION_CREATE_RECEIVED:
+    /* We got the 'CREATE' (incoming connection), should send the CREATE_ACK */
+    cc->metrics.age = GNUNET_TIME_absolute_get ();
+    cc->task = GNUNET_SCHEDULER_add_now (&send_create_ack,
+                                         cc);
+    break;
+  case CADET_CONNECTION_READY:
+    if ( (NULL == cc->keepalive_qe) &&
+         (GNUNET_YES == cc->mqm_ready) &&
+         (NULL == cc->task) )
     {
-      GNUNET_break (GNUNET_NO != c->destroy);
-      return;
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Scheduling keepalive for %s in %s\n",
+           GCC_2s (cc),
+           GNUNET_STRINGS_relative_time_to_string (keepalive_period,
+                                                   GNUNET_YES));
+      cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                               &send_keepalive,
+                                               cc);
     }
-    GCT_handle_encrypted (c->t, msg);
-    GCC_send_ack (c, fwd, GNUNET_NO);
-    GCC_check_connections ();
-    return;
+    break;
   }
-
-  /* Message not for us: forward to next hop */
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  not for us, retransmitting...\n");
-  GNUNET_STATISTICS_update (stats, "# messages forwarded", 1, GNUNET_NO);
-  (void) GCC_send_prebuilt_message (&msg->header, 0,
-                                    zero, c, fwd,
-                                    GNUNET_NO, NULL, NULL);
-  GCC_check_connections ();
 }
 
 
 /**
- * Initialize the connections subsystem
+ * Create a connection to @a destination via @a path and notify @a cb
+ * whenever we are ready for more data.  Shared logic independent of
+ * who is initiating the connection.
  *
- * @param c Configuration handle.
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param off offset of @a destination on @a path
+ * @param options options for the connection
+ * @param ct which tunnel uses this connection
+ * @param init_state initial state for the connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection
  */
-void
-GCC_init (const struct GNUNET_CONFIGURATION_Handle *c)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "init\n");
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c, "CADET", "MAX_MSGS_QUEUE",
-                                             &max_msgs_queue))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
-                               "CADET", "MAX_MSGS_QUEUE", "MISSING");
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c, "CADET", "MAX_CONNECTIONS",
-                                             &max_connections))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
-                               "CADET", "MAX_CONNECTIONS", "MISSING");
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c, "CADET", "REFRESH_CONNECTION_TIME",
-                                           &refresh_connection_time))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
-                               "CADET", "REFRESH_CONNECTION_TIME", "MISSING");
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  create_connection_time = GNUNET_TIME_relative_min (GNUNET_TIME_UNIT_SECONDS,
-                                                     refresh_connection_time);
-  connections = GNUNET_CONTAINER_multishortmap_create (1024,
-                                                       GNUNET_YES);
-}
-
-
-/**
- * Destroy each connection on shutdown.
- *
- * @param cls Closure (unused).
- * @param key Current key code (CID, unused).
- * @param value Value in the hash map (`struct CadetConnection`)
- *
- * @return #GNUNET_YES, because we should continue to iterate
- */
-static int
-shutdown_iterator (void *cls,
-                   const struct GNUNET_ShortHashCode *key,
-                   void *value)
-{
-  struct CadetConnection *c = value;
-
-  c->state = CADET_CONNECTION_DESTROYED;
-  GCC_destroy (c);
-  return GNUNET_YES;
-}
-
-
-/**
- * Shut down the connections subsystem.
- */
-void
-GCC_shutdown (void)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Shutting down connections\n");
-  GCC_check_connections ();
-  GNUNET_CONTAINER_multishortmap_iterate (connections,
-                                          &shutdown_iterator,
-                                          NULL);
-  GNUNET_CONTAINER_multishortmap_destroy (connections);
-  connections = NULL;
-}
-
-
-/**
- * Create a connection.
- *
- * @param cid Connection ID (either created locally or imposed remotely).
- * @param t Tunnel this connection belongs to (or NULL for transit connections);
- * @param path Path this connection has to use (copy is made).
- * @param own_pos Own position in the @c path path.
- *
- * @return Newly created connection.
- *         NULL in case of error: own id not in path, wrong neighbors, ...
-*/
-struct CadetConnection *
-GCC_new (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-         struct CadetTunnel *t,
-         struct CadetPeerPath *path,
-         unsigned int own_pos)
-{
-  struct CadetConnection *c;
-  struct CadetPeerPath *cpath;
-
-  GCC_check_connections ();
-  cpath = path_duplicate (path);
-  GNUNET_assert (NULL != cpath);
-  c = GNUNET_new (struct CadetConnection);
-  c->id = *cid;
+static struct CadetConnection *
+connection_create (struct CadetPeer *destination,
+                   struct CadetPeerPath *path,
+                   unsigned int off,
+                   enum GNUNET_CADET_ChannelOption options,
+                   struct CadetTConnection *ct,
+                   const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                   enum CadetConnectionState init_state,
+                   GCC_ReadyCallback ready_cb,
+                   void *ready_cb_cls)
+{
+  struct CadetConnection *cc;
+  struct CadetPeer *first_hop;
+
+  cc = GNUNET_new (struct CadetConnection);
+  cc->options = options;
+  cc->state = init_state;
+  cc->ct = ct;
+  cc->cid = *cid;
   GNUNET_assert (GNUNET_OK ==
                  GNUNET_CONTAINER_multishortmap_put (connections,
-                                                     &c->id.connection_of_tunnel,
-                                                     c,
+                                                     &GCC_get_id (cc)->connection_of_tunnel,
+                                                     cc,
                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  fc_init (&c->fwd_fc);
-  fc_init (&c->bck_fc);
-  c->fwd_fc.c = c;
-  c->bck_fc.c = c;
-
-  c->t = t;
-  GNUNET_assert (own_pos <= cpath->length - 1);
-  c->own_pos = own_pos;
-  c->path = cpath;
-  cpath->c = c;
-  if (GNUNET_OK != register_neighbors (c))
-  {
-    if (0 == own_pos)
-    {
-      /* We were the origin of this request, this means we have invalid
-       * info about the paths to reach the destination. We must invalidate
-       * the *original* path to avoid trying it again in the next minute.
-       */
-      if (2 < path->length)
-        path_invalidate (path);
-      else
-      {
-        GNUNET_break (0);
-        GCT_debug(t, GNUNET_ERROR_TYPE_WARNING);
-      }
-      c->t = NULL;
-    }
-    path_destroy (c->path);
-    c->path = NULL;
-    GCC_destroy (c);
-    return NULL;
-  }
-  LOG (GNUNET_ERROR_TYPE_INFO, "New connection %s\n", GCC_2s (c));
-  GCC_check_connections ();
-  return c;
-}
-
-
-/**
- * Connection is no longer needed: destroy it.
- *
- * Cancels all pending traffic (including possible DESTROY messages), all
- * maintenance tasks and removes the connection from neighbor peers and tunnel.
- *
- * @param c Connection to destroy.
- */
-void
-GCC_destroy (struct CadetConnection *c)
-{
-  GCC_check_connections ();
-  if (NULL == c)
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  if (2 == c->destroy) /* cancel queues -> GCP_queue_cancel -> q_destroy -> */
-    return;            /* -> message_sent -> GCC_destroy. Don't loop. */
-  c->destroy = 2;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "destroying connection %s\n",
-       GCC_2s (c));
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " fc's f: %p, b: %p\n",
-       &c->fwd_fc, &c->bck_fc);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " fc tasks f: %u, b: %u\n",
-       c->fwd_fc.poll_task,
-       c->bck_fc.poll_task);
-
-  /* Cancel all traffic */
-  if (NULL != c->path)
-  {
-    connection_cancel_queues (c, GNUNET_YES);
-    connection_cancel_queues (c, GNUNET_NO);
-    if (NULL != c->maintenance_q)
-    {
-      GCP_send_cancel (c->maintenance_q);
-      c->maintenance_q = NULL;
-    }
-  }
-  unregister_neighbors (c);
-  path_destroy (c->path);
-  c->path = NULL;
-
-  /* Delete from tunnel */
-  if (NULL != c->t)
-    GCT_remove_connection (c->t, c);
-
-  if (NULL != c->check_duplicates_task)
-    GNUNET_SCHEDULER_cancel (c->check_duplicates_task);
-  if (NULL != c->fwd_maintenance_task)
-    GNUNET_SCHEDULER_cancel (c->fwd_maintenance_task);
-  if (NULL != c->bck_maintenance_task)
-    GNUNET_SCHEDULER_cancel (c->bck_maintenance_task);
-
-  if (GNUNET_NO == c->was_removed)
-  {
-    GNUNET_break (GNUNET_YES ==
-                  GNUNET_CONTAINER_multishortmap_remove (connections,
-                                                         &c->id.connection_of_tunnel,
-                                                         c));
-  }
-  GNUNET_STATISTICS_update (stats,
-                            "# connections",
-                            -1,
-                            GNUNET_NO);
-  GNUNET_free (c);
-  GCC_check_connections ();
-}
-
-
-/**
- * Get the connection ID.
- *
- * @param c Connection to get the ID from.
- *
- * @return ID of the connection.
- */
-const struct GNUNET_CADET_ConnectionTunnelIdentifier *
-GCC_get_id (const struct CadetConnection *c)
-{
-  return &c->id;
-}
-
-
-/**
- * Get the connection path.
- *
- * @param c Connection to get the path from.
- *
- * @return path used by the connection.
- */
-const struct CadetPeerPath *
-GCC_get_path (const struct CadetConnection *c)
-{
-  if (GNUNET_NO == c->destroy)
-    return c->path;
-  return NULL;
-}
-
-
-/**
- * Get the connection state.
- *
- * @param c Connection to get the state from.
- *
- * @return state of the connection.
- */
-enum CadetConnectionState
-GCC_get_state (const struct CadetConnection *c)
-{
-  return c->state;
-}
-
-/**
- * Get the connection tunnel.
- *
- * @param c Connection to get the tunnel from.
- *
- * @return tunnel of the connection.
- */
-struct CadetTunnel *
-GCC_get_tunnel (const struct CadetConnection *c)
-{
-  return c->t;
-}
-
-
-/**
- * Get free buffer space in a connection.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- *
- * @return Free buffer space [0 - max_msgs_queue/max_connections]
- */
-unsigned int
-GCC_get_buffer (struct CadetConnection *c, int fwd)
-{
-  struct CadetFlowControl *fc;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  Get %s buffer on %s: %u - %u\n",
-       GC_f2s (fwd), GCC_2s (c), fc->queue_max, fc->queue_n);
-  GCC_debug (c, GNUNET_ERROR_TYPE_DEBUG);
-
-  return (fc->queue_max - fc->queue_n);
-}
-
-
-/**
- * Get how many messages have we allowed to send to us from a direction.
- *
- * @param c Connection.
- * @param fwd Are we asking about traffic from FWD (BCK messages)?
- *
- * @return last_ack_sent - last_pid_recv
- */
-unsigned int
-GCC_get_allowed (struct CadetConnection *c, int fwd)
-{
-  struct CadetFlowControl *fc;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  if ( (CADET_CONNECTION_READY != c->state) ||
-       GC_is_pid_bigger (ntohl (fc->last_pid_recv.pid),
-                         ntohl (fc->last_ack_sent.pid)) )
-  {
-    return 0;
-  }
-  return (ntohl (fc->last_ack_sent.pid) - ntohl (fc->last_pid_recv.pid));
-}
-
-
-/**
- * Get messages queued in a connection.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- *
- * @return Number of messages queued.
- */
-unsigned int
-GCC_get_qn (struct CadetConnection *c, int fwd)
-{
-  struct CadetFlowControl *fc;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-
-  return fc->queue_n;
-}
-
-
-/**
- * Get next PID to use.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- * @return Next PID to use.
- */
-struct CadetEncryptedMessageIdentifier
-GCC_get_pid (struct CadetConnection *c, int fwd)
-{
-  struct CadetFlowControl *fc;
-  struct CadetEncryptedMessageIdentifier pid;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  pid = fc->next_pid;
-  fc->next_pid.pid = htonl (1 + ntohl (pid.pid));
-  return pid;
-}
-
-
-/**
- * Allow the connection to advertise a buffer of the given size.
- *
- * The connection will send an @c fwd ACK message (so: in direction !fwd)
- * allowing up to last_pid_recv + buffer.
- *
- * @param c Connection.
- * @param buffer How many more messages the connection can accept.
- * @param fwd Is this about FWD traffic? (The ack will go dest->root).
- */
-void
-GCC_allow (struct CadetConnection *c, unsigned int buffer, int fwd)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  allowing %s %u messages %s\n",
-       GCC_2s (c), buffer, GC_f2s (fwd));
-  send_ack (c, buffer, fwd, GNUNET_NO);
-}
-
-
-/**
- * Notify other peers on a connection of a broken link. Mark connections
- * to destroy after all traffic has been sent.
- *
- * @param c Connection on which there has been a disconnection.
- * @param peer Peer that disconnected.
- */
-void
-GCC_neighbor_disconnected (struct CadetConnection *c, struct CadetPeer *peer)
-{
-  struct CadetFlowControl *fc;
-  char peer_name[16];
-  int fwd;
-
-  GCC_check_connections ();
-  strncpy (peer_name, GCP_2s (peer), 16);
-  peer_name[15] = '\0';
+  cc->ready_cb = ready_cb;
+  cc->ready_cb_cls = ready_cb_cls;
+  cc->path = path;
+  cc->off = off;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "shutting down %s, %s disconnected\n",
-       GCC_2s (c), peer_name);
-
-  invalidate_paths (c, peer);
-
-  fwd = is_fwd (c, peer);
-  if (GNUNET_SYSERR == fwd)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if ( (GNUNET_YES == GCC_is_terminal (c, fwd)) ||
-       (GNUNET_NO != c->destroy) )
-  {
-    /* Local shutdown, or other peer already down (hence 'c->destroy');
-       so there is no one to notify about this, just clean up. */
-    GCC_destroy (c);
-    GCC_check_connections ();
-    return;
-  }
-  /* Mark FlowControl towards the peer as unavaliable. */
-  fc = fwd ? &c->bck_fc : &c->fwd_fc;
-  fc->queue_max = 0;
-
-  send_broken (c, &my_full_id, GCP_get_id (peer), fwd);
-
-  /* Connection will have at least one pending message
-   * (the one we just scheduled), so delay destruction
-   * and remove from map so we don't use accidentally. */
-  mark_destroyed (c);
-  GNUNET_assert (GNUNET_NO == c->was_removed);
-  c->was_removed = GNUNET_YES;
-  GNUNET_break (GNUNET_YES ==
-                GNUNET_CONTAINER_multishortmap_remove (connections,
-                                                       &c->id.connection_of_tunnel,
-                                                       c));
-  /* Cancel queue in the direction that just died. */
-  connection_cancel_queues (c, ! fwd);
-  GCC_stop_poll (c, ! fwd);
-  unregister_neighbors (c);
-  GCC_check_connections ();
-}
-
-
-/**
- * Is this peer the first one on the connection?
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *
- * @return #GNUNET_YES if origin, #GNUNET_NO if relay/terminal.
+       "Creating %s using path %s\n",
+       GCC_2s (cc),
+       GCPP_2s (path));
+  GCPP_add_connection (path,
+                       off,
+                       cc);
+  for (unsigned int i=0;i<off;i++)
+    GCP_add_connection (GCPP_get_peer_at_offset (path,
+                                                 i),
+                        cc);
+
+  first_hop = GCPP_get_peer_at_offset (path,
+                                       0);
+  cc->mq_man = GCP_request_mq (first_hop,
+                               &manage_first_hop_mq,
+                               cc);
+  return cc;
+}
+
+
+/**
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.  This
+ * is an inbound tunnel, so we must use the existing @a cid
+ *
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param options options for the connection
+ * @param ct which tunnel uses this connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection, NULL if we already have
+ *         a connection that takes precedence on @a path
  */
-int
-GCC_is_origin (struct CadetConnection *c, int fwd)
-{
-  if (!fwd && c->path->length - 1 == c->own_pos )
-    return GNUNET_YES;
-  if (fwd && 0 == c->own_pos)
-    return GNUNET_YES;
-  return GNUNET_NO;
-}
-
-
-/**
- * Is this peer the last one on the connection?
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *            Note that the ROOT is the terminal for BCK traffic!
- *
- * @return #GNUNET_YES if terminal, #GNUNET_NO if relay/origin.
- */
-int
-GCC_is_terminal (struct CadetConnection *c, int fwd)
-{
-  return GCC_is_origin (c, ! fwd);
-}
-
-
-/**
- * See if we are allowed to send by the next hop in the given direction.
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *
- * @return #GNUNET_YES in case it's OK to send.
- */
-int
-GCC_is_sendable (struct CadetConnection *c, int fwd)
-{
-  struct CadetFlowControl *fc;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " checking sendability of %s traffic on %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-  if (NULL == c)
-  {
-    GNUNET_break (0);
-    return GNUNET_YES;
-  }
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " last ack recv: %u, last pid sent: %u\n",
-       ntohl (fc->last_ack_recv.pid),
-       ntohl (fc->last_pid_sent.pid));
-  if (GC_is_pid_bigger (ntohl (fc->last_ack_recv.pid),
-                        ntohl (fc->last_pid_sent.pid)))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " sendable\n");
-    return GNUNET_YES;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " not sendable\n");
-  return GNUNET_NO;
-}
-
-
-/**
- * Check if this connection is a direct one (never trim a direct connection).
- *
- * @param c Connection.
- *
- * @return #GNUNET_YES in case it's a direct connection, #GNUNET_NO otherwise.
- */
-int
-GCC_is_direct (struct CadetConnection *c)
-{
-  return (c->path->length == 2) ? GNUNET_YES : GNUNET_NO;
-}
-
-
-/**
- * Sends a completely built message on a connection, properly registering
- * all used resources.
- *
- * @param message Message to send.
- * @param payload_type Type of payload, in case the message is encrypted.
- *                     0 for restransmissions (when type is no longer known)
- *                     UINT16_MAX when not applicable.
- * @param payload_id ID of the payload (PID, ACK, ...).
- * @param c Connection on which this message is transmitted.
- * @param fwd Is this a fwd message?
- * @param force Force the connection to accept the message (buffer overfill).
- * @param cont Continuation called once message is sent. Can be NULL.
- * @param cont_cls Closure for @c cont.
- *
- * @return Handle to cancel the message before it's sent.
- *         NULL on error.
- *         Invalid on @c cont call.
- */
-struct CadetConnectionQueue *
-GCC_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                           uint16_t payload_type,
-                           struct CadetEncryptedMessageIdentifier payload_id,
-                           struct CadetConnection *c, int fwd, int force,
-                           GCC_sent cont, void *cont_cls)
-{
-  struct CadetFlowControl *fc;
-  struct CadetConnectionQueue *q;
-  uint16_t size;
-  uint16_t type;
-
-  size = ntohs (message->size);
-  type = ntohs (message->type);
-
-  GCC_check_connections ();
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  if (0 == fc->queue_max)
-  {
-    GNUNET_break (0);
-    return NULL;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "--> %s (%s %4u) on conn %s (%p) %s [%5u]\n",
-       GC_m2s (type), GC_m2s (payload_type), payload_id, GCC_2s (c), c,
-       GC_f2s(fwd), size);
-  switch (type)
-  {
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED:
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  Q_N+ %p %u, PIDsnt: %u, ACKrcv: %u\n",
-           fc,
-           fc->queue_n,
-           ntohl (fc->last_pid_sent.pid),
-           ntohl (fc->last_ack_recv.pid));
-      if (GNUNET_NO == force)
-      {
-        fc->queue_n++;
-      }
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX:
-      /* nothing to do here */
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK:
-       /* Should've only be used for restransmissions. */
-      GNUNET_break (0 == payload_type);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK:
-    case GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY:
-    case GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN:
-      GNUNET_assert (GNUNET_YES == force);
-      break;
-
-    default:
-      GNUNET_break (0);
+struct CadetConnection *
+GCC_create_inbound (struct CadetPeer *destination,
+                    struct CadetPeerPath *path,
+                    enum GNUNET_CADET_ChannelOption options,
+                    struct CadetTConnection *ct,
+                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                    GCC_ReadyCallback ready_cb,
+                    void *ready_cb_cls)
+{
+  struct CadetConnection *cc;
+  unsigned int off;
+
+  off = GCPP_find_peer (path,
+                        destination);
+  GNUNET_assert (UINT_MAX != off);
+  cc = GCPP_get_connection (path,
+                            destination,
+                            off);
+  if (NULL != cc)
+  {
+    int cmp;
+
+    cmp = memcmp (cid,
+                  &cc->cid,
+                  sizeof (*cid));
+    if (0 == cmp)
+    {
+      /* Two peers picked the SAME random connection identifier at the
+         same time for the same path? Must be malicious.  Drop
+         connection (existing and inbound), even if it is the only
+         one. */
+      GNUNET_break_op (0);
+      GCT_connection_lost (cc->ct);
+      GCC_destroy_without_tunnel (cc);
       return NULL;
-  }
-
-  if (fc->queue_n > fc->queue_max && GNUNET_NO == force)
-  {
-    GNUNET_STATISTICS_update (stats, "# messages dropped (buffer full)",
-                              1, GNUNET_NO);
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "queue full: %u/%u\n",
-         fc->queue_n, fc->queue_max);
-    if (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED == type)
+    }
+    if (0 < cmp)
     {
-      fc->queue_n--;
+      /* drop existing */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got two connections on %s, dropping my existing %s\n",
+           GCPP_2s (path),
+           GCC_2s (cc));
+      GCT_connection_lost (cc->ct);
+      GCC_destroy_without_tunnel (cc);
     }
-    return NULL; /* Drop this message */
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  C_P+ %s %u\n",
-       GCC_2s (c), c->pending_messages);
-  c->pending_messages++;
-
-  q = GNUNET_new (struct CadetConnectionQueue);
-  q->cont = cont;
-  q->cont_cls = cont_cls;
-  q->forced = force;
-  GNUNET_CONTAINER_DLL_insert (fc->q_head, fc->q_tail, q);
-  q->peer_q = GCP_send (get_hop (c, fwd),
-                        message,
-                        payload_type,
-                        payload_id,
-                        c,
-                        fwd,
-                        &conn_message_sent, q);
-  if (NULL == q->peer_q)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "dropping msg on %s, NULL q\n", GCC_2s (c));
-    GNUNET_CONTAINER_DLL_remove (fc->q_head, fc->q_tail, q);
-    GNUNET_free (q);
-    GCC_check_connections ();
-    return NULL;
-  }
-  GCC_check_connections ();
-  return q;
-}
-
-
-/**
- * Cancel a previously sent message while it's in the queue.
- *
- * ONLY can be called before the continuation given to the send function
- * is called. Once the continuation is called, the message is no longer in the
- * queue.
- *
- * @param q Handle to the queue.
- */
-void
-GCC_cancel (struct CadetConnectionQueue *q)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "!  GCC cancel message\n");
-
-  /* send_cancel calls message_sent, which calls q->cont and frees q */
-  GCP_send_cancel (q->peer_q);
-  GCC_check_connections ();
-}
-
-
-/**
- * Sends a CREATE CONNECTION message for a path to a peer.
- * Changes the connection and tunnel states if necessary.
- *
- * @param c Connection to create.
- */
-void
-GCC_send_create (struct CadetConnection *c)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  enum CadetTunnelCState state;
-  size_t size;
-
-  GCC_check_connections ();
-  size = sizeof (struct GNUNET_CADET_ConnectionCreateMessage);
-  size += c->path->length * sizeof (struct GNUNET_PeerIdentity);
-  {
-    /* Allocate message on the stack */
-    unsigned char cbuf[size];
-    struct GNUNET_CADET_ConnectionCreateMessage *msg;
-    struct GNUNET_PeerIdentity *peers;
-
-
-    msg = (struct GNUNET_CADET_ConnectionCreateMessage *) cbuf;
-    msg->header.size = htons (size);
-    msg->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE);
-    msg->options = htonl (0);
-    msg->cid = *GCC_get_id (c);
-    peers = (struct GNUNET_PeerIdentity *) &msg[1];
-    for (int i = 0; i < c->path->length; i++)
+    else
     {
-      GNUNET_PEER_resolve (c->path->peers[i], peers++);
+      /* keep existing */
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got two connections on %s, keeping my existing %s\n",
+           GCPP_2s (path),
+           GCC_2s (cc));
+      return NULL;
     }
-    GNUNET_assert (NULL == c->maintenance_q);
-    c->maintenance_q = GCP_send (get_next_hop (c),
-                                 &msg->header,
-                                 GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE,
-                                 zero,
-                                 c, GNUNET_YES,
-                                 &conn_message_sent, NULL);
   }
 
-  LOG (GNUNET_ERROR_TYPE_INFO, "==> %s %19s on conn %s (%p) FWD [%5u]\n",
-       GC_m2s (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE), "",
-       GCC_2s (c), c, size);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  C_P+ %p %u (create)\n",
-         c, c->pending_messages);
-  c->pending_messages++;
-
-  state = GCT_get_cstate (c->t);
-  if (CADET_TUNNEL_SEARCHING == state || CADET_TUNNEL_NEW == state)
-    GCT_change_cstate (c->t, CADET_TUNNEL_WAITING);
-  if (CADET_CONNECTION_NEW == c->state)
-    connection_change_state (c, CADET_CONNECTION_SENT);
-  GCC_check_connections ();
+  return connection_create (destination,
+                            path,
+                            off,
+                            options,
+                            ct,
+                            cid,
+                            CADET_CONNECTION_CREATE_RECEIVED,
+                            ready_cb,
+                            ready_cb_cls);
 }
 
 
 /**
- * Send an ACK on the appropriate connection/channel, depending on
- * the direction and the position of the peer.
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.
  *
- * @param c Which connection to send the hop-by-hop ACK.
- * @param fwd Is this a fwd ACK? (will go dest->root).
- * @param force Send the ACK even if suboptimal (e.g. requested by POLL).
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param off offset of @a destination on @a path
+ * @param options options for the connection
+ * @param ct tunnel that uses the connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection
  */
-void
-GCC_send_ack (struct CadetConnection *c, int fwd, int force)
+struct CadetConnection *
+GCC_create (struct CadetPeer *destination,
+            struct CadetPeerPath *path,
+            unsigned int off,
+            enum GNUNET_CADET_ChannelOption options,
+            struct CadetTConnection *ct,
+            GCC_ReadyCallback ready_cb,
+            void *ready_cb_cls)
 {
-  unsigned int buffer;
-
-  GCC_check_connections ();
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GCC send %s ACK on %s\n",
-       GC_f2s (fwd), GCC_2s (c));
-
-  if (NULL == c)
-  {
-    GNUNET_break (0);
-    return;
-  }
+  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
 
-  if (GNUNET_NO != c->destroy)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  being destroyed, why bother...\n");
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Get available buffer space */
-  if (GCC_is_terminal (c, fwd))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  getting from all channels\n");
-    buffer = GCT_get_channels_buffer (c->t);
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  getting from one connection\n");
-    buffer = GCC_get_buffer (c, fwd);
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  buffer available: %u\n", buffer);
-  if (0 == buffer && GNUNET_NO == force)
-  {
-    GCC_check_connections ();
-    return;
-  }
-
-  /* Send available buffer space */
-  if (GNUNET_YES == GCC_is_origin (c, fwd))
-  {
-    GNUNET_assert (NULL != c->t);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  sending on channels...\n");
-    GCT_unchoke_channels (c->t);
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  sending on connection\n");
-    send_ack (c, buffer, fwd, force);
-  }
-  GCC_check_connections ();
+  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_NONCE,
+                              &cid,
+                              sizeof (cid));
+  return connection_create (destination,
+                            path,
+                            off,
+                            options,
+                            ct,
+                            &cid,
+                            CADET_CONNECTION_NEW,
+                            ready_cb,
+                            ready_cb_cls);
 }
 
 
 /**
- * Send a message to all peers in this connection that the connection
- * is no longer valid.
+ * Transmit message @a msg via connection @a cc.  Must only be called
+ * (once) after the connection has signalled that it is ready via the
+ * `ready_cb`.  Clients can also use #GCC_is_ready() to check if the
+ * connection is right now ready for transmission.
  *
- * If some peer should not receive the message, it should be zero'ed out
- * before calling this function.
- *
- * @param c The connection whose peers to notify.
+ * @param cc connection identification
+ * @param env envelope with message to transmit; must NOT
+ *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
  */
 void
-GCC_send_destroy (struct CadetConnection *c)
+GCC_transmit (struct CadetConnection *cc,
+              struct GNUNET_MQ_Envelope *env)
 {
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct GNUNET_CADET_ConnectionDestroyMessage msg;
-
-  if (GNUNET_YES == c->destroy)
-    return;
-  GCC_check_connections ();
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY);
-  msg.cid = c->id;
-  msg.reserved = htonl (0);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-              "  sending connection destroy for connection %s\n",
-              GCC_2s (c));
-
-  if (GNUNET_NO == GCC_is_terminal (c, GNUNET_YES))
-    (void) GCC_send_prebuilt_message (&msg.header,
-                                      UINT16_MAX,
-                                      zero,
-                                      c,
-                                      GNUNET_YES, GNUNET_YES, NULL, NULL);
-  if (GNUNET_NO == GCC_is_terminal (c, GNUNET_NO))
-    (void) GCC_send_prebuilt_message (&msg.header,
-                                      UINT16_MAX,
-                                      zero,
-                                      c,
-                                      GNUNET_NO, GNUNET_YES, NULL, NULL);
-  mark_destroyed (c);
-  GCC_check_connections ();
+       "Scheduling message for transmission on %s\n",
+       GCC_2s (cc));
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  GNUNET_assert (CADET_CONNECTION_READY == cc->state);
+  cc->metrics.last_use = GNUNET_TIME_absolute_get ();
+  cc->mqm_ready = GNUNET_NO;
+  if (NULL != cc->task)
+  {
+    GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = NULL;
+  }
+  GCP_send (cc->mq_man,
+            env);
 }
 
 
 /**
- * @brief Start a polling timer for the connection.
- *
- * When a neighbor does not accept more traffic on the connection it could be
- * caused by a simple congestion or by a lost ACK. Polling enables to check
- * for the lastest ACK status for a connection.
+ * Obtain the path used by this connection.
  *
- * @param c Connection.
- * @param fwd Should we poll in the FWD direction?
+ * @param cc connection
+ * @return path to @a cc
  */
-void
-GCC_start_poll (struct CadetConnection *c, int fwd)
+struct CadetPeerPath *
+GCC_get_path (struct CadetConnection *cc)
 {
-  struct CadetFlowControl *fc;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "POLL %s requested\n",
-       GC_f2s (fwd));
-  if (NULL != fc->poll_task || NULL != fc->poll_msg)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  POLL already in progress (t: %p, m: %p)\n",
-         fc->poll_task, fc->poll_msg);
-    return;
-  }
-  if (0 == fc->queue_max)
-  {
-    /* Should not be needed, traffic should've been cancelled. */
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  POLL not possible, peer disconnected\n");
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "POLL started on request\n");
-  fc->poll_task = GNUNET_SCHEDULER_add_delayed (fc->poll_time, &send_poll, fc);
+  return cc->path;
 }
 
 
 /**
- * @brief Stop polling a connection for ACKs.
+ * Obtain unique ID for the connection.
  *
- * Once we have enough ACKs for future traffic, polls are no longer necessary.
- *
- * @param c Connection.
- * @param fwd Should we stop the poll in the FWD direction?
+ * @param cc connection.
+ * @return unique number of the connection
  */
-void
-GCC_stop_poll (struct CadetConnection *c, int fwd)
+const struct GNUNET_CADET_ConnectionTunnelIdentifier *
+GCC_get_id (struct CadetConnection *cc)
 {
-  struct CadetFlowControl *fc;
-
-  fc = fwd ? &c->fwd_fc : &c->bck_fc;
-  if (NULL != fc->poll_task)
-  {
-    GNUNET_SCHEDULER_cancel (fc->poll_task);
-    fc->poll_task = NULL;
-  }
-  if (NULL != fc->poll_msg)
-  {
-    GCC_cancel (fc->poll_msg);
-    fc->poll_msg = NULL;
-  }
+  return &cc->cid;
 }
 
 
 /**
  * Get a (static) string for a connection.
  *
- * @param c Connection.
+ * @param cc Connection.
  */
 const char *
-GCC_2s (const struct CadetConnection *c)
+GCC_2s (const struct CadetConnection *cc)
 {
-  if (NULL == c)
-    return "NULL";
+  static char buf[128];
 
-  if (NULL != c->t)
-  {
-    static char buf[128];
+  if (NULL == cc)
+    return "Connection(NULL)";
 
-    SPRINTF (buf, "%s (->%s)",
-             GNUNET_sh2s (&GCC_get_id (c)->connection_of_tunnel),
-             GCT_2s (c->t));
+  if (NULL != cc->ct)
+  {
+    GNUNET_snprintf (buf,
+                     sizeof (buf),
+                     "Connection %s (%s)",
+                     GNUNET_sh2s (&cc->cid.connection_of_tunnel),
+                     GCT_2s (cc->ct->t));
     return buf;
   }
-  return GNUNET_sh2s (&c->id.connection_of_tunnel);
+  GNUNET_snprintf (buf,
+                   sizeof (buf),
+                   "Connection %s",
+                   GNUNET_sh2s (&cc->cid.connection_of_tunnel));
+  return buf;
 }
 
 
+#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-con",__VA_ARGS__)
+
+
 /**
- * Log all possible info about the connection state.
+ * Log connection info.
  *
- * @param c Connection to debug.
+ * @param cc connection
  * @param level Debug level to use.
  */
 void
-GCC_debug (const struct CadetConnection *c, enum GNUNET_ErrorType level)
+GCC_debug (struct CadetConnection *cc,
+           enum GNUNET_ErrorType level)
 {
   int do_log;
-  char *s;
 
   do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
                                        "cadet-con",
                                        __FILE__, __FUNCTION__, __LINE__);
   if (0 == do_log)
     return;
-
-  if (NULL == c)
+  if (NULL == cc)
   {
-    LOG2 (level, "CCC DEBUG NULL CONNECTION\n");
+    LOG2 (level,
+          "Connection (NULL)\n");
     return;
   }
-
-  LOG2 (level, "CCC DEBUG CONNECTION %s\n", GCC_2s (c));
-  s = path_2s (c->path);
-  LOG2 (level, "CCC  path %s, own pos: %u\n", s, c->own_pos);
-  GNUNET_free (s);
-  LOG2 (level, "CCC  state: %s, destroy: %u\n",
-        GCC_state2s (c->state), c->destroy);
-  LOG2 (level, "CCC  pending messages: %u\n", c->pending_messages);
-  if (NULL != c->perf)
-    LOG2 (level, "CCC  us/byte: %f\n", c->perf->avg);
-
-  LOG2 (level, "CCC  FWD flow control:\n");
-  LOG2 (level, "CCC   queue: %u/%u\n", c->fwd_fc.queue_n, c->fwd_fc.queue_max);
-  LOG2 (level, "CCC   last PID sent: %5u, recv: %5u\n",
-        ntohl (c->fwd_fc.last_pid_sent.pid),
-        ntohl (c->fwd_fc.last_pid_recv.pid));
-  LOG2 (level, "CCC   last ACK sent: %5u, recv: %5u\n",
-        ntohl (c->fwd_fc.last_ack_sent.pid),
-        ntohl (c->fwd_fc.last_ack_recv.pid));
-  LOG2 (level, "CCC   recv PID bitmap: %X\n", c->fwd_fc.recv_bitmap);
-  LOG2 (level, "CCC   poll: task %d, msg  %p, msg_ack %p)\n",
-        c->fwd_fc.poll_task, c->fwd_fc.poll_msg, c->fwd_fc.ack_msg);
-
-  LOG2 (level, "CCC  BCK flow control:\n");
-  LOG2 (level, "CCC   queue: %u/%u\n", c->bck_fc.queue_n, c->bck_fc.queue_max);
-  LOG2 (level, "CCC   last PID sent: %5u, recv: %5u\n",
-        ntohl (c->bck_fc.last_pid_sent.pid),
-        ntohl (c->bck_fc.last_pid_recv.pid));
-  LOG2 (level, "CCC   last ACK sent: %5u, recv: %5u\n",
-        ntohl (c->bck_fc.last_ack_sent.pid),
-        ntohl (c->bck_fc.last_ack_recv.pid));
-  LOG2 (level, "CCC   recv PID bitmap: %X\n", c->bck_fc.recv_bitmap);
-  LOG2 (level, "CCC   poll: task %d, msg  %p, msg_ack %p)\n",
-        c->bck_fc.poll_task, c->bck_fc.poll_msg, c->bck_fc.ack_msg);
-
-  LOG2 (level, "CCC DEBUG CONNECTION END\n");
+  LOG2 (level,
+        "%s to %s via path %s in state %d is %s\n",
+        GCC_2s (cc),
+        GCP_2s (cc->destination),
+        GCPP_2s (cc->path),
+        cc->state,
+        (GNUNET_YES == cc->mqm_ready) ? "ready" : "busy");
 }
+
+/* end of gnunet-service-cadet-new_connection.c */
diff --git a/src/cadet/gnunet-service-cadet_connection.h b/src/cadet/gnunet-service-cadet_connection.h
index 307cb42c2..fdb184366 100644
--- a/src/cadet/gnunet-service-cadet_connection.h
+++ b/src/cadet/gnunet-service-cadet_connection.h
@@ -1,6 +1,7 @@
+
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -20,557 +21,319 @@
 
 /**
  * @file cadet/gnunet-service-cadet_connection.h
- * @brief cadet service; dealing with connections
+ * @brief A connection is a live end-to-end messaging mechanism
+ *       where the peers are identified by a path and know how
+ *       to forward along the route using a connection identifier
+ *       for routing the data.
  * @author Bartlomiej Polot
- *
- * All functions in this file use the prefix GCC (GNUnet Cadet Connection)
+ * @author Christian Grothoff
  */
-
 #ifndef GNUNET_SERVICE_CADET_CONNECTION_H
 #define GNUNET_SERVICE_CADET_CONNECTION_H
 
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
 #include "gnunet_util_lib.h"
-
-
-/**
- * All the states a connection can be in.
- */
-enum CadetConnectionState
-{
-  /**
-   * Uninitialized status, should never appear in operation.
-   */
-  CADET_CONNECTION_NEW,
-
-  /**
-   * Connection create message sent, waiting for ACK.
-   */
-  CADET_CONNECTION_SENT,
-
-  /**
-   * Connection ACK sent, waiting for ACK.
-   */
-  CADET_CONNECTION_ACK,
-
-  /**
-   * Connection confirmed, ready to carry traffic.
-   */
-  CADET_CONNECTION_READY,
-
-  /**
-   * Connection to be destroyed, just waiting to empty queues.
-   */
-  CADET_CONNECTION_DESTROYED,
-
-  /**
-   * Connection to be destroyed because of a distant peer, same as DESTROYED.
-   */
-  CADET_CONNECTION_BROKEN,
-};
-
-
-/**
- * Struct containing all information regarding a connection to a peer.
- */
-struct CadetConnection;
-
-/**
- * Handle for messages queued but not yet sent.
- */
-struct CadetConnectionQueue;
-
-#include "cadet_path.h"
-#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet.h"
 #include "gnunet-service-cadet_peer.h"
+#include "cadet_protocol.h"
 
 
 /**
- * Check invariants for all connections using #check_neighbours().
- */
-void
-GCC_check_connections (void);
-
-
-/**
- * Callback called when a queued message is sent.
+ * Function called to notify tunnel about change in our readyness.
  *
- * @param cls Closure.
- * @param c Connection this message was on.
- * @param type Type of message sent.
- * @param fwd Was this a FWD going message?
- * @param size Size of the message.
+ * @param cls closure
+ * @param is_ready #GNUNET_YES if the connection is now ready for transmission,
+ *                 #GNUNET_NO if the connection is no longer ready for transmission
  */
 typedef void
-(*GCC_sent) (void *cls,
-             struct CadetConnection *c,
-             struct CadetConnectionQueue *q,
-             uint16_t type,
-             int fwd,
-             size_t size);
+(*GCC_ReadyCallback)(void *cls,
+                     int is_ready);
 
 
 /**
- * Handler for connection creation.
+ * Destroy a connection, called when the CORE layer is already done
+ * (i.e. has received a BROKEN message), but if we still have to
+ * communicate the destruction of the connection to the tunnel (if one
+ * exists).
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection to destroy
  */
 void
-GCC_handle_create (struct CadetPeer *peer,
-                   const struct GNUNET_CADET_ConnectionCreateMessage *msg);
+GCC_destroy_without_core (struct CadetConnection *cc);
 
 
 /**
- * Handler for connection confirmations.
+ * Destroy a connection, called if the tunnel association with the
+ * connection was already broken, but we still need to notify the CORE
+ * layer about the breakage.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection to destroy
  */
 void
-GCC_handle_confirm (struct CadetPeer *peer,
-                    const struct GNUNET_CADET_ConnectionCreateAckMessage *msg);
+GCC_destroy_without_tunnel (struct CadetConnection *cc);
 
 
 /**
- * Handler for notifications of broken connections.
+ * Lookup a connection by its identifier.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cid identifier to resolve
+ * @return NULL if connection was not found
  */
-void
-GCC_handle_broken (struct CadetPeer *peer,
-                   const struct GNUNET_CADET_ConnectionBrokenMessage *msg);
+struct CadetConnection *
+GCC_lookup (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
 
-/**
- * Handler for notifications of destroyed connections.
- *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
- */
-void
-GCC_handle_destroy (struct CadetPeer *peer,
-                    const struct GNUNET_CADET_ConnectionDestroyMessage *msg);
 
 /**
- * Handler for cadet network traffic hop-by-hop acks.
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param off offset of @a destination on @a path
+ * @param options options for the connection
+ * @param ct which tunnel uses this connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection
  */
-void
-GCC_handle_ack (struct CadetPeer *peer,
-                const struct GNUNET_CADET_ConnectionEncryptedAckMessage *msg);
+struct CadetConnection *
+GCC_create (struct CadetPeer *destination,
+            struct CadetPeerPath *path,
+            unsigned int off,
+            enum GNUNET_CADET_ChannelOption options,
+            struct CadetTConnection *ct,
+            GCC_ReadyCallback ready_cb,
+            void *ready_cb_cls);
 
-/**
- * Handler for cadet network traffic hop-by-hop data counter polls.
- *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
- */
-void
-GCC_handle_poll (struct CadetPeer *peer,
-                 const struct GNUNET_CADET_ConnectionHopByHopPollMessage *msg);
 
 /**
- * Handler for key exchange traffic (Axolotl KX).
+ * Create a connection to @a destination via @a path and
+ * notify @a cb whenever we are ready for more data.  This
+ * is an inbound tunnel, so we must use the existing @a cid
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param destination where to go
+ * @param path which path to take (may not be the full path)
+ * @param options options for the connection
+ * @param ct which tunnel uses this connection
+ * @param ready_cb function to call when ready to transmit
+ * @param ready_cb_cls closure for @a cb
+ * @return handle to the connection, NULL if we already have
+ *         a connection that takes precedence on @a path
  */
-void
-GCC_handle_kx (struct CadetPeer *peer,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg);
+struct CadetConnection *
+GCC_create_inbound (struct CadetPeer *destination,
+                    struct CadetPeerPath *path,
+                    enum GNUNET_CADET_ChannelOption options,
+                    struct CadetTConnection *ct,
+                    const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                    GCC_ReadyCallback ready_cb,
+                    void *ready_cb_cls);
+
 
 /**
- * Handler for encrypted cadet network traffic (channel mgmt, data).
+ * Transmit message @a msg via connection @a cc.  Must only be called
+ * (once) after the connection has signalled that it is ready via the
+ * `ready_cb`.  Clients can also use #GCC_is_ready() to check if the
+ * connection is right now ready for transmission.
  *
- * @param peer Message sender (neighbor).
- * @param msg Message itself.
+ * @param cc connection identification
+ * @param env envelope with message to transmit;
+ *            the #GNUNET_MQ_notify_send() must not have yet been used
+ *            for the envelope.  Also, the message better match the
+ *            connection identifier of this connection...
  */
 void
-GCC_handle_encrypted (struct CadetPeer *peer,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg);
+GCC_transmit (struct CadetConnection *cc,
+              struct GNUNET_MQ_Envelope *env);
 
-/**
- * Core handler for axolotl key exchange traffic.
- *
- * @param cls Closure (unused).
- * @param message Message received.
- * @param peer Neighbor who sent the message.
- *
- * @return GNUNET_OK, to keep the connection open.
- */
-int
-GCC_handle_ax_kx (void *cls, const struct GNUNET_PeerIdentity *peer,
-                  const struct GNUNET_MessageHeader *message);
 
 /**
- * Core handler for axolotl encrypted cadet network traffic.
+ * A CREATE_ACK was received for this connection, process it.
  *
- * @param cls Closure (unused).
- * @param message Message received.
- * @param peer Neighbor who sent the message.
- *
- * @return GNUNET_OK, to keep the connection open.
+ * @param cc the connection that got the ACK.
  */
-int
-GCC_handle_ax (void *cls, const struct GNUNET_PeerIdentity *peer,
-               struct GNUNET_MessageHeader *message);
+void
+GCC_handle_connection_create_ack (struct CadetConnection *cc);
 
-/**
- * Core handler for cadet keepalives.
- *
- * @param cls closure
- * @param message message
- * @param peer peer identity this notification is about
- * @return GNUNET_OK to keep the connection open,
- *         GNUNET_SYSERR to close it (signal serious error)
- *
- * TODO: Check who we got this from, to validate route.
- */
-int
-GCC_handle_keepalive (void *cls, const struct GNUNET_PeerIdentity *peer,
-                      const struct GNUNET_MessageHeader *message);
 
 /**
- * Send an ACK on the appropriate connection/channel, depending on
- * the direction and the position of the peer.
+ * We got a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE for a
+ * connection that we already have.  Either our ACK got lost
+ * or something is fishy.  Consider retransmitting the ACK.
  *
- * @param c Which connection to send the hop-by-hop ACK.
- * @param fwd Is this a fwd ACK? (will go dest->root).
- * @param force Send the ACK even if suboptimal (e.g. requested by POLL).
+ * @param cc connection that got the duplicate CREATE
  */
 void
-GCC_send_ack (struct CadetConnection *c, int fwd, int force);
+GCC_handle_duplicate_create (struct CadetConnection *cc);
 
-/**
- * Initialize the connections subsystem
- *
- * @param c Configuration handle.
- */
-void
-GCC_init (const struct GNUNET_CONFIGURATION_Handle *c);
 
 /**
- * Shut down the connections subsystem.
+ * Handle KX message.
+ *
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
  */
 void
-GCC_shutdown (void);
+GCC_handle_kx (struct CadetConnection *cc,
+               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg);
 
-/**
- * Create a connection.
- *
- * @param cid Connection ID (either created locally or imposed remotely).
- * @param t Tunnel this connection belongs to (or NULL for transit connections);
- * @param path Path this connection has to use (copy is made).
- * @param own_pos Own position in the @c path path.
- *
- * @return Newly created connection.
- *         NULL in case of error: own id not in path, wrong neighbors, ...
- */
-struct CadetConnection *
-GCC_new (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
-         struct CadetTunnel *t,
-         struct CadetPeerPath *path,
-         unsigned int own_pos);
 
 /**
- * Connection is no longer needed: destroy it.
- *
- * Cancels all pending traffic (including possible DESTROY messages), all
- * maintenance tasks and removes the connection from neighbor peers and tunnel.
+ * Handle KX_AUTH message.
  *
- * @param c Connection to destroy.
+ * @param cc connection that received encrypted message
+ * @param msg the key exchange message
  */
 void
-GCC_destroy (struct CadetConnection *c);
-
-/**
- * Get the connection ID.
- *
- * @param c Connection to get the ID from.
- *
- * @return ID of the connection.
- */
-const struct GNUNET_CADET_ConnectionTunnelIdentifier *
-GCC_get_id (const struct CadetConnection *c);
+GCC_handle_kx_auth (struct CadetConnection *cc,
+                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg);
 
 
 /**
- * Get the connection path.
- *
- * @param c Connection to get the path from.
- *
- * @return path used by the connection.
+ * Performance metrics for a connection.
  */
-const struct CadetPeerPath *
-GCC_get_path (const struct CadetConnection *c);
+struct CadetConnectionMetrics
+{
 
-/**
- * Get the connection state.
- *
- * @param c Connection to get the state from.
- *
- * @return state of the connection.
- */
-enum CadetConnectionState
-GCC_get_state (const struct CadetConnection *c);
+  /**
+   * Our current best estimate of the latency, based on a weighted
+   * average of at least @a latency_datapoints values.
+   */
+  struct GNUNET_TIME_Relative aged_latency;
 
-/**
- * Get the connection tunnel.
- *
- * @param c Connection to get the tunnel from.
- *
- * @return tunnel of the connection.
- */
-struct CadetTunnel *
-GCC_get_tunnel (const struct CadetConnection *c);
+  /**
+   * When was this connection first established? (by us sending or
+   * receiving the CREATE_ACK for the first time)
+   */
+  struct GNUNET_TIME_Absolute age;
 
-/**
- * Get free buffer space in a connection.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- *
- * @return Free buffer space [0 - max_msgs_queue/max_connections]
- */
-unsigned int
-GCC_get_buffer (struct CadetConnection *c, int fwd);
+  /**
+   * When was this connection last used? (by us sending or
+   * receiving a PAYLOAD message on it)
+   */
+  struct GNUNET_TIME_Absolute last_use;
 
-/**
- * Get how many messages have we allowed to send to us from a direction.
- *
- * @param c Connection.
- * @param fwd Are we asking about traffic from FWD (BCK messages)?
- *
- * @return last_ack_sent - last_pid_recv
- */
-unsigned int
-GCC_get_allowed (struct CadetConnection *c, int fwd);
+  /**
+   * How many packets that ought to generate an ACK did we send via
+   * this connection?
+   */
+  unsigned long long num_acked_transmissions;
 
-/**
- * Get messages queued in a connection.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- *
- * @return Number of messages queued.
- */
-unsigned int
-GCC_get_qn (struct CadetConnection *c, int fwd);
+  /**
+   * Number of packets that were sent via this connection did actually
+   * receive an ACK?  (Note: ACKs may be transmitted and lost via
+   * other connections, so this value should only be interpreted
+   * relative to @e num_acked_transmissions and in relation to other
+   * connections.)
+   */
+  unsigned long long num_successes;
 
-/**
- * Get next PID to use.
- *
- * @param c Connection.
- * @param fwd Is query about FWD traffic?
- * @return Next PID to use.
- */
-struct CadetEncryptedMessageIdentifier
-GCC_get_pid (struct CadetConnection *c, int fwd);
+};
 
-/**
- * Allow the connection to advertise a buffer of the given size.
- *
- * The connection will send an @c fwd ACK message (so: in direction !fwd)
- * allowing up to last_pid_recv + buffer.
- *
- * @param c Connection.
- * @param buffer How many more messages the connection can accept.
- * @param fwd Is this about FWD traffic? (The ack will go dest->root).
- */
-void
-GCC_allow (struct CadetConnection *c, unsigned int buffer, int fwd);
 
 /**
- * Send FWD keepalive packets for a connection.
+ * Obtain performance @a metrics from @a cc.
  *
- * @param cls Closure (connection for which to send the keepalive).
- * @param tc Notification context.
+ * @param cc connection to query
+ * @return the metrics
  */
-void
-GCC_fwd_keepalive (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+const struct CadetConnectionMetrics *
+GCC_get_metrics (struct CadetConnection *cc);
+
 
 /**
- * Send BCK keepalive packets for a connection.
+ * Handle encrypted message.
  *
- * @param cls Closure (connection for which to send the keepalive).
- * @param tc Notification context.
+ * @param cc connection that received encrypted message
+ * @param msg the encrypted message to decrypt
  */
 void
-GCC_bck_keepalive (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+GCC_handle_encrypted (struct CadetConnection *cc,
+                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg);
 
 
 /**
- * Notify other peers on a connection of a broken link. Mark connections
- * to destroy after all traffic has been sent.
+ * We sent a message for which we expect to receive an ACK via
+ * the connection identified by @a cti.
  *
- * @param c Connection on which there has been a disconnection.
- * @param peer Peer that disconnected.
+ * @param cid connection identifier where we expect an ACK
  */
 void
-GCC_neighbor_disconnected (struct CadetConnection *c, struct CadetPeer *peer);
+GCC_ack_expected (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
 
-/**
- * Is this peer the first one on the connection?
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *
- * @return #GNUNET_YES if origin, #GNUNET_NO if relay/terminal.
- */
-int
-GCC_is_origin (struct CadetConnection *c, int fwd);
 
 /**
- * Is this peer the last one on the connection?
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *            Note that the ROOT is the terminal for BCK traffic!
+ * We observed an ACK for a message that was originally sent via
+ * the connection identified by @a cti.
  *
- * @return #GNUNET_YES if terminal, #GNUNET_NO if relay/origin.
+ * @param cid connection identifier where we got an ACK for a message
+ *            that was originally sent via this connection (the ACK
+ *            may have gotten back to us via a different connection).
  */
-int
-GCC_is_terminal (struct CadetConnection *c, int fwd);
-
-/**
- * See if we are allowed to send by the next hop in the given direction.
- *
- * @param c Connection.
- * @param fwd Is this about fwd traffic?
- *
- * @return #GNUNET_YES in case it's OK to send.
- */
-int
-GCC_is_sendable (struct CadetConnection *c, int fwd);
+void
+GCC_ack_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid);
 
-/**
- * Check if this connection is a direct one (never trim a direct connection).
- *
- * @param c Connection.
- *
- * @return #GNUNET_YES in case it's a direct connection, #GNUNET_NO otherwise.
- */
-int
-GCC_is_direct (struct CadetConnection *c);
 
 /**
- * Cancel a previously sent message while it's in the queue.
+ * We observed some the given @a latency on the connection
+ * identified by @a cti.  (The same connection was taken
+ * in both directions.)
  *
- * ONLY can be called before the continuation given to the send function
- * is called. Once the continuation is called, the message is no longer in the
- * queue.
- *
- * @param q Handle to the queue.
+ * @param cti connection identifier where we measured latency
+ * @param latency the observed latency
  */
 void
-GCC_cancel (struct CadetConnectionQueue *q);
+GCC_latency_observed (const struct GNUNET_CADET_ConnectionTunnelIdentifier *cti,
+                      struct GNUNET_TIME_Relative latency);
 
-/**
- * Sends an already built message on a connection, properly registering
- * all used resources.
- *
- * @param message Message to send.
- * @param payload_type Type of payload, in case the message is encrypted.
- *                     0 for restransmissions (when type is no longer known)
- *                     UINT16_MAX when not applicable.
- * @param payload_id ID of the payload (PID, ACK, ...).
- * @param c Connection on which this message is transmitted.
- * @param fwd Is this a fwd message?
- * @param force Force the connection to accept the message (buffer overfill).
- * @param cont Continuation called once message is sent. Can be NULL.
- * @param cont_cls Closure for @c cont.
- *
- * @return Handle to cancel the message before it's sent.
- *         NULL on error.
- *         Invalid on @c cont call.
- */
-struct CadetConnectionQueue *
-GCC_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                           uint16_t payload_type,
-                           struct CadetEncryptedMessageIdentifier payload_id,
-                           struct CadetConnection *c, int fwd, int force,
-                           GCC_sent cont, void *cont_cls);
 
 /**
- * Sends a CREATE CONNECTION message for a path to a peer.
- * Changes the connection and tunnel states if necessary.
+ * Return the tunnel associated with this connection.
  *
- * @param connection Connection to create.
+ * @param cc connection to query
+ * @return corresponding entry in the tunnel's connection list
  */
-void
-GCC_send_create (struct CadetConnection *connection);
+struct CadetTConnection *
+GCC_get_ct (struct CadetConnection *cc);
 
-/**
- * Send a message to all peers in this connection that the connection
- * is no longer valid.
- *
- * If some peer should not receive the message, it should be zero'ed out
- * before calling this function.
- *
- * @param c The connection whose peers to notify.
- */
-void
-GCC_send_destroy (struct CadetConnection *c);
 
 /**
- * @brief Start a polling timer for the connection.
+ * Obtain the path used by this connection.
  *
- * When a neighbor does not accept more traffic on the connection it could be
- * caused by a simple congestion or by a lost ACK. Polling enables to check
- * for the lastest ACK status for a connection.
- *
- * @param c Connection.
- * @param fwd Should we poll in the FWD direction?
+ * @param cc connection
+ * @return path to @a cc
  */
-void
-GCC_start_poll (struct CadetConnection *c, int fwd);
+struct CadetPeerPath *
+GCC_get_path (struct CadetConnection *cc);
 
 
 /**
- * @brief Stop polling a connection for ACKs.
- *
- * Once we have enough ACKs for future traffic, polls are no longer necessary.
+ * Obtain unique ID for the connection.
  *
- * @param c Connection.
- * @param fwd Should we stop the poll in the FWD direction?
+ * @param cc connection.
+ * @return unique number of the connection
  */
-void
-GCC_stop_poll (struct CadetConnection *c, int fwd);
+const struct GNUNET_CADET_ConnectionTunnelIdentifier *
+GCC_get_id (struct CadetConnection *cc);
+
 
 /**
  * Get a (static) string for a connection.
  *
- * @param c Connection.
+ * @param cc Connection.
  */
 const char *
-GCC_2s (const struct CadetConnection *c);
+GCC_2s (const struct CadetConnection *cc);
+
 
 /**
- * Log all possible info about the connection state.
+ * Log connection info.
  *
- * @param c Connection to debug.
+ * @param cc connection
  * @param level Debug level to use.
  */
 void
-GCC_debug (const struct CadetConnection *c, enum GNUNET_ErrorType level);
+GCC_debug (struct CadetConnection *cc,
+           enum GNUNET_ErrorType level);
 
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
 
-/* ifndef GNUNET_SERVICE_CADET_CONNECTION_H */
 #endif
-/* end of gnunet-service-cadet_connection.h */
diff --git a/src/cadet/gnunet-service-cadet-new_core.c b/src/cadet/gnunet-service-cadet_core.c
index 3768c36a5..ae03b4f35 100644
--- a/src/cadet/gnunet-service-cadet-new_core.c
+++ b/src/cadet/gnunet-service-cadet_core.c
@@ -30,11 +30,11 @@
  * - Optimization: given BROKEN messages, destroy paths (?)
  */
 #include "platform.h"
-#include "gnunet-service-cadet-new_core.h"
-#include "gnunet-service-cadet-new_paths.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_tunnels.h"
+#include "gnunet-service-cadet_core.h"
+#include "gnunet-service-cadet_paths.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
 #include "gnunet_core_service.h"
 #include "gnunet_statistics_service.h"
 #include "cadet_protocol.h"
diff --git a/src/cadet/gnunet-service-cadet-new_core.h b/src/cadet/gnunet-service-cadet_core.h
index 65b0a6ba5..65b0a6ba5 100644
--- a/src/cadet/gnunet-service-cadet-new_core.h
+++ b/src/cadet/gnunet-service-cadet_core.h
diff --git a/src/cadet/gnunet-service-cadet_dht.c b/src/cadet/gnunet-service-cadet_dht.c
index 22673b167..f00c0caf3 100644
--- a/src/cadet/gnunet-service-cadet_dht.c
+++ b/src/cadet/gnunet-service-cadet_dht.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2013, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,25 +17,41 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
-
+/**
+ * @file cadet/gnunet-service-cadet_dht.c
+ * @brief Information we track per peer.
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ */
 
 #include "platform.h"
 #include "gnunet_util_lib.h"
-
 #include "gnunet_dht_service.h"
 #include "gnunet_statistics_service.h"
-
-#include "cadet_path.h"
+#include "gnunet-service-cadet.h"
 #include "gnunet-service-cadet_dht.h"
-#include "gnunet-service-cadet_peer.h"
 #include "gnunet-service-cadet_hello.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
 
-#define LOG(level, ...) GNUNET_log_from (level,"cadet-dht",__VA_ARGS__)
+/**
+ * How long do we wait before first announcing our presence to the DHT.
+ * Used to wait for our HELLO to be available.  Note that we also get
+ * notifications when our HELLO is ready, so this is just the maximum
+ * we wait for the first notification.
+ */
+#define STARTUP_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 500)
 
+/**
+ * How long do we wait after we get an updated HELLO before publishing?
+ * Allows for the HELLO to be updated again quickly, for example in
+ * case multiple addresses changed and we got a partial update.
+ */
+#define CHANGE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 100)
+
+
+#define LOG(level, ...) GNUNET_log_from (level,"cadet-dht",__VA_ARGS__)
 
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
 
 /**
  * Handle for DHT searches.
@@ -47,42 +63,9 @@ struct GCD_search_handle
    */
   struct GNUNET_DHT_GetHandle *dhtget;
 
-  /**
-   * Provided callback to call when a path is found.
-   */
-  GCD_search_callback callback;
-
-  /**
-   * Provided closure.
-   */
-  void *cls;
-
-  /**
-   * Peer ID searched for
-   */
-  GNUNET_PEER_Id peer_id;
 };
 
 
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Own ID (short value).
- */
-extern GNUNET_PEER_Id myid;
-
-/**
- * Own ID (full value).
- */
-extern struct GNUNET_PeerIdentity my_full_id;
-
 /**
  * Handle to use DHT.
  */
@@ -94,69 +77,20 @@ static struct GNUNET_DHT_Handle *dht_handle;
 static struct GNUNET_TIME_Relative id_announce_time;
 
 /**
- * DHT replication level, see DHT API: GNUNET_DHT_get_start, GNUNET_DHT_put.
+ * DHT replication level, see DHT API: #GNUNET_DHT_get_start(), #GNUNET_DHT_put().
  */
 static unsigned long long dht_replication_level;
 
 /**
  * Task to periodically announce itself in the network.
  */
-static struct GNUNET_SCHEDULER_Task * announce_id_task;
+static struct GNUNET_SCHEDULER_Task *announce_id_task;
 
 /**
  * Delay for the next ID announce.
  */
 static struct GNUNET_TIME_Relative announce_delay;
 
-/**
- * GET requests to stop on shutdown.
- */
-static struct GNUNET_CONTAINER_MultiHashMap32 *get_requests;
-
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
-
-/**
- * Build a PeerPath from the paths returned from the DHT, reversing the paths
- * to obtain a local peer -> destination path and interning the peer ids.
- *
- * @return Newly allocated and created path
- *
- * FIXME refactor and use build_path_from_peer_ids
- */
-static struct CadetPeerPath *
-path_build_from_dht (const struct GNUNET_PeerIdentity *get_path,
-                     unsigned int get_path_length,
-                     const struct GNUNET_PeerIdentity *put_path,
-                     unsigned int put_path_length)
-{
-  size_t size = get_path_length + put_path_length + 1;
-  struct GNUNET_PeerIdentity peers[size];
-  const struct GNUNET_PeerIdentity *peer;
-  struct CadetPeerPath *p;
-  unsigned int own_pos;
-  int i;
-
-  peers[0] = my_full_id;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "   GET has %d hops.\n", get_path_length);
-  for (i = 0 ; i < get_path_length; i++)
-  {
-    peer = &get_path[get_path_length - i - 1];
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "   From GET: %s\n", GNUNET_i2s (peer));
-    peers[i + 1] = *peer;
-  }
-  for (i = 0 ; i < put_path_length; i++)
-  {
-    peer = &put_path[put_path_length - i - 1];
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "   From PUT: %s\n", GNUNET_i2s (peer));
-    peers[i + get_path_length + 1] = *peer;
-  }
-  p = path_build_from_peer_ids (peers, size, myid, &own_pos);
-  return p;
-}
-
 
 /**
  * Function to process paths received for a new peer addition. The recorded
@@ -176,42 +110,34 @@ path_build_from_dht (const struct GNUNET_PeerIdentity *get_path,
  */
 static void
 dht_get_id_handler (void *cls, struct GNUNET_TIME_Absolute exp,
-                    const struct GNUNET_HashCode * key,
+                    const struct GNUNET_HashCode *key,
                     const struct GNUNET_PeerIdentity *get_path,
                     unsigned int get_path_length,
                     const struct GNUNET_PeerIdentity *put_path,
-                    unsigned int put_path_length, enum GNUNET_BLOCK_Type type,
-                    size_t size, const void *data)
+                    unsigned int put_path_length,
+                    enum GNUNET_BLOCK_Type type,
+                    size_t size,
+                    const void *data)
 {
-  struct GCD_search_handle *h = cls;
-  struct GNUNET_HELLO_Message *hello;
-  struct CadetPeerPath *p;
+  const struct GNUNET_HELLO_Message *hello = data;
   struct CadetPeer *peer;
-  char *s;
 
-  p = path_build_from_dht (get_path, get_path_length,
-                           put_path, put_path_length);
-  if (NULL == p)
+  GCPP_try_path_from_dht (get_path,
+                          get_path_length,
+                          put_path,
+                          put_path_length);
+  if ( (size >= sizeof (struct GNUNET_HELLO_Message)) &&
+       (ntohs (hello->header.size) == size) &&
+       (size == GNUNET_HELLO_size (hello)) )
   {
-    GNUNET_break_op (0);
-    return;
+    peer = GCP_get (&put_path[0],
+                    GNUNET_YES);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got HELLO for %s\n",
+         GCP_2s (peer));
+    GCP_set_hello (peer,
+                   hello);
   }
-
-  s = path_2s (p);
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "Got path from DHT: %s\n",
-       s);
-  GNUNET_free_non_null (s);
-
-  peer = GCP_get_short (p->peers[p->length - 1], GNUNET_YES);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Got HELLO for %s\n",
-       GCP_2s (peer));
-  h->callback (h->cls, p);
-  path_destroy (p);
-  hello = (struct GNUNET_HELLO_Message *) data;
-  GCP_set_hello (peer, hello);
-  GCP_try_connect (peer);
 }
 
 
@@ -229,19 +155,10 @@ announce_id (void *cls)
   struct GNUNET_TIME_Absolute expiration;
   struct GNUNET_TIME_Relative next_put;
 
-  announce_id_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Announce ID\n");
   hello = GCH_get_mine ();
   size = (NULL != hello) ? GNUNET_HELLO_size (hello) : 0;
-  if ( (NULL == hello) || (0 == size) )
+  if (0 == size)
   {
-    /* Peerinfo gave us no hello yet, try again soon. */
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         "  no hello, waiting!\n");
-    GNUNET_STATISTICS_update (stats,
-                              "# DHT announce skipped (no hello)",
-                              1,
-                              GNUNET_NO);
     expiration = GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get (),
                                            announce_delay);
     announce_delay = GNUNET_TIME_STD_BACKOFF (announce_delay);
@@ -252,71 +169,64 @@ announce_id (void *cls)
     announce_delay = GNUNET_TIME_UNIT_SECONDS;
   }
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Hello %p size: %u\n",
-       hello,
-       size);
-  if (NULL != hello)
-  {
-    GNUNET_STATISTICS_update (stats,
-                              "# DHT announce",
-                              1, GNUNET_NO);
-    memset (&phash,
-            0,
-            sizeof (phash));
-    GNUNET_memcpy (&phash,
-                   &my_full_id,
-                   sizeof (my_full_id));
-    GNUNET_DHT_put (dht_handle,   /* DHT handle */
-                    &phash,       /* Key to use */
-                    dht_replication_level,     /* Replication level */
-                    GNUNET_DHT_RO_RECORD_ROUTE
-                    | GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,    /* DHT options */
-                    GNUNET_BLOCK_TYPE_DHT_HELLO,       /* Block type */
-                    size,  /* Size of the data */
-                    (const char *) hello, /* Data itself */
-                    expiration,  /* Data expiration */
-                    NULL,         /* Continuation */
-                    NULL);        /* Continuation closure */
-  }
   /* Call again in id_announce_time, unless HELLO expires first,
    * but wait at least 1s. */
-  next_put = GNUNET_TIME_absolute_get_remaining (expiration);
-  next_put = GNUNET_TIME_relative_min (next_put,
-                                       id_announce_time);
-  next_put = GNUNET_TIME_relative_max (next_put,
-                                       GNUNET_TIME_UNIT_SECONDS);
-  announce_id_task = GNUNET_SCHEDULER_add_delayed (next_put,
-                                                   &announce_id,
-                                                   cls);
+  next_put
+    = GNUNET_TIME_absolute_get_remaining (expiration);
+  next_put
+    = GNUNET_TIME_relative_min (next_put,
+                                id_announce_time);
+  next_put
+    = GNUNET_TIME_relative_max (next_put,
+                                GNUNET_TIME_UNIT_SECONDS);
+  announce_id_task
+    = GNUNET_SCHEDULER_add_delayed (next_put,
+                                    &announce_id,
+                                    cls);
+  GNUNET_STATISTICS_update (stats,
+                            "# DHT announce",
+                            1,
+                            GNUNET_NO);
+  memset (&phash,
+          0,
+          sizeof (phash));
+  GNUNET_memcpy (&phash,
+                 &my_full_id,
+                 sizeof (my_full_id));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Announcing my HELLO (%u bytes) in the DHT\n",
+       size);
+  GNUNET_DHT_put (dht_handle,   /* DHT handle */
+                  &phash,       /* Key to use */
+                  dht_replication_level,     /* Replication level */
+                  GNUNET_DHT_RO_RECORD_ROUTE
+                  | GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,    /* DHT options */
+                  GNUNET_BLOCK_TYPE_DHT_HELLO,       /* Block type */
+                  size,  /* Size of the data */
+                  (const char *) hello, /* Data itself */
+                  expiration,  /* Data expiration */
+                  NULL,         /* Continuation */
+                  NULL);        /* Continuation closure */
 }
 
+
 /**
- * Iterator over hash map entries and stop GET requests before disconnecting
- * from the DHT.
- *
- * @param cls Closure (unused)
- * @param key Current peer ID.
- * @param value Value in the hash map (GCD_search_handle).
- *
- * @return #GNUNET_YES, we should continue to iterate,
+ * Function called by the HELLO subsystem whenever OUR hello
+ * changes. Re-triggers the DHT PUT immediately.
  */
-int
-stop_get (void *cls,
-          uint32_t key,
-          void *value)
+void
+GCD_hello_update ()
 {
-  struct GCD_search_handle *h = value;
-
-  GCD_search_stop (h);
-  return GNUNET_YES;
+  if (NULL == announce_id_task)
+    return; /* too early */
+  GNUNET_SCHEDULER_cancel (announce_id_task);
+  announce_id_task
+    = GNUNET_SCHEDULER_add_delayed (CHANGE_DELAY,
+                                    &announce_id,
+                                    NULL);
 }
 
 
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
 /**
  * Initialize the DHT subsystem.
  *
@@ -325,36 +235,40 @@ stop_get (void *cls,
 void
 GCD_init (const struct GNUNET_CONFIGURATION_Handle *c)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "init\n");
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c, "CADET",
+      GNUNET_CONFIGURATION_get_value_number (c,
+                                             "CADET",
                                              "DHT_REPLICATION_LEVEL",
                                              &dht_replication_level))
   {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING, "CADET",
-                               "DHT_REPLICATION_LEVEL", "USING DEFAULT");
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "DHT_REPLICATION_LEVEL",
+                               "USING DEFAULT");
     dht_replication_level = 3;
   }
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c, "CADET", "ID_ANNOUNCE_TIME",
+      GNUNET_CONFIGURATION_get_value_time (c,
+                                           "CADET",
+                                           "ID_ANNOUNCE_TIME",
                                            &id_announce_time))
   {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR, "CADET",
-                               "ID_ANNOUNCE_TIME", "MISSING");
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
+                               "CADET",
+                               "ID_ANNOUNCE_TIME",
+                               "MISSING");
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
 
-  dht_handle = GNUNET_DHT_connect (c, 64);
-  if (NULL == dht_handle)
-  {
-    GNUNET_break (0);
-  }
-
+  dht_handle = GNUNET_DHT_connect (c,
+                                   64);
+  GNUNET_break (NULL != dht_handle);
   announce_delay = GNUNET_TIME_UNIT_SECONDS;
-  announce_id_task = GNUNET_SCHEDULER_add_now (&announce_id, NULL);
-  get_requests = GNUNET_CONTAINER_multihashmap32_create (32);
+  announce_id_task = GNUNET_SCHEDULER_add_delayed (STARTUP_DELAY,
+                                                   &announce_id,
+                                                   NULL);
 }
 
 
@@ -364,10 +278,7 @@ GCD_init (const struct GNUNET_CONFIGURATION_Handle *c)
 void
 GCD_shutdown (void)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Shutting down DHT\n");
-  GNUNET_CONTAINER_multihashmap32_iterate (get_requests, &stop_get, NULL);
-  GNUNET_CONTAINER_multihashmap32_destroy (get_requests);
-  if (dht_handle != NULL)
+  if (NULL != dht_handle)
   {
     GNUNET_DHT_disconnect (dht_handle);
     dht_handle = NULL;
@@ -379,22 +290,31 @@ GCD_shutdown (void)
   }
 }
 
+
+/**
+ * Search DHT for paths to @a peeR_id
+ *
+ * @param peer_id peer to search for
+ * @return handle to abort search
+ */
 struct GCD_search_handle *
-GCD_search (const struct GNUNET_PeerIdentity *peer_id,
-            GCD_search_callback callback, void *cls)
+GCD_search (const struct GNUNET_PeerIdentity *peer_id)
 {
   struct GNUNET_HashCode phash;
   struct GCD_search_handle *h;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Starting DHT GET for peer %s\n",
-       GNUNET_i2s (peer_id));
-  GNUNET_STATISTICS_update (stats, "# DHT search", 1, GNUNET_NO);
-  memset (&phash, 0, sizeof (phash));
-  GNUNET_memcpy (&phash, peer_id, sizeof (*peer_id));
+  GNUNET_STATISTICS_update (stats,
+                            "# DHT search",
+                            1,
+                            GNUNET_NO);
+  memset (&phash,
+          0,
+          sizeof (phash));
+  GNUNET_memcpy (&phash,
+                 peer_id,
+                 sizeof (*peer_id));
+
   h = GNUNET_new (struct GCD_search_handle);
-  h->peer_id = GNUNET_PEER_intern (peer_id);
-  h->callback = callback;
-  h->cls = cls;
   h->dhtget = GNUNET_DHT_get_start (dht_handle,    /* handle */
                                     GNUNET_BLOCK_TYPE_DHT_HELLO, /* type */
                                     &phash,     /* key to search */
@@ -405,20 +325,27 @@ GCD_search (const struct GNUNET_PeerIdentity *peer_id,
                                     0,     /* xquery bits */
                                     &dht_get_id_handler,
                                     h);
-  GNUNET_CONTAINER_multihashmap32_put (get_requests,
-                                       h->peer_id,
-                                       h,
-                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Starting DHT GET for peer %s (%p)\n",
+       GNUNET_i2s (peer_id),
+       h);
   return h;
 }
 
 
+/**
+ * Stop DHT search started with #GCD_search().
+ *
+ * @param h handle to search to stop
+ */
 void
 GCD_search_stop (struct GCD_search_handle *h)
 {
-  GNUNET_break (GNUNET_OK ==
-                GNUNET_CONTAINER_multihashmap32_remove (get_requests,
-                                                        h->peer_id, h));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Stopping DHT GET %p\n",
+       h);
   GNUNET_DHT_get_stop (h->dhtget);
   GNUNET_free (h);
 }
+
+/* end of gnunet-service-cadet_dht.c */
diff --git a/src/cadet/gnunet-service-cadet_dht.h b/src/cadet/gnunet-service-cadet_dht.h
index b70dfe975..5d7ab29a0 100644
--- a/src/cadet/gnunet-service-cadet_dht.h
+++ b/src/cadet/gnunet-service-cadet_dht.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2013, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -22,10 +22,10 @@
  * @file cadet/gnunet-service-cadet_dht.h
  * @brief cadet service; dealing with DHT requests and results
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
  *
- * All functions in this file should use the prefix GMD (Gnunet Cadet Dht)
+ * All functions in this file should use the prefix GCD (Gnunet Cadet Dht)
  */
-
 #ifndef GNUNET_SERVICE_CADET_DHT_H
 #define GNUNET_SERVICE_CADET_DHT_H
 
@@ -40,23 +40,11 @@ extern "C"
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-struct GCD_search_handle;
-
-
 /**
- * Callback called on each path found over the DHT.
- *
- * @param cls Closure.
- * @param path An unchecked, unoptimized path to the target node.
- *             After callback will no longer be valid!
+ * Handle for DHT search operation.
  */
-typedef void
-(*GCD_search_callback) (void *cls,
-                        const struct CadetPeerPath *path);
+struct GCD_search_handle;
 
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
 
 /**
  * Initialize the DHT subsystem.
@@ -66,6 +54,7 @@ typedef void
 void
 GCD_init (const struct GNUNET_CONFIGURATION_Handle *c);
 
+
 /**
  * Shut down the DHT subsystem.
  */
@@ -73,14 +62,32 @@ void
 GCD_shutdown (void);
 
 
+/**
+ * Function called by the HELLO subsystem whenever OUR hello
+ * changes. Re-triggers the DHT PUT immediately.
+ */
+void
+GCD_hello_update (void);
+
+/**
+ * Search DHT for paths to @a peeR_id
+ *
+ * @param peer_id peer to search for
+ * @return handle to abort search
+ */
 struct GCD_search_handle *
-GCD_search (const struct GNUNET_PeerIdentity *peer_id,
-            GCD_search_callback callback, void *cls);
+GCD_search (const struct GNUNET_PeerIdentity *peer_id);
 
 
+/**
+ * Stop DHT search started with #GCD_search().
+ *
+ * @param h handle to search to stop
+ */
 void
 GCD_search_stop (struct GCD_search_handle *h);
 
+
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
 #endif
@@ -88,6 +95,6 @@ GCD_search_stop (struct GCD_search_handle *h);
 }
 #endif
 
-/* ifndef GNUNET_CADET_SERVICE_LOCAL_H */
+/* ifndef GNUNET_CADET_SERVICE_DHT_H */
 #endif
-/* end of gnunet-cadet-service_LOCAL.h */
+/* end of gnunet-service-cadet_dht.h */
diff --git a/src/cadet/gnunet-service-cadet_hello.c b/src/cadet/gnunet-service-cadet_hello.c
index 3c63f3551..6d85de39f 100644
--- a/src/cadet/gnunet-service-cadet_hello.c
+++ b/src/cadet/gnunet-service-cadet_hello.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2014 GNUnet e.V.
+     Copyright (C) 2014, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,58 +17,33 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
-
+/**
+ * @file cadet/gnunet-service-cadet_hello.c
+ * @brief spread knowledge about how to contact other peers from PEERINFO
+ * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - is most of this necessary/helpful?
+ * - should we not simply restrict this to OUR hello?
+ */
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
 #include "gnunet_statistics_service.h"
 #include "gnunet_peerinfo_service.h"
-
 #include "cadet_protocol.h"
-#include "cadet_path.h"
-
+#include "gnunet-service-cadet.h"
+#include "gnunet-service-cadet_dht.h"
 #include "gnunet-service-cadet_hello.h"
 #include "gnunet-service-cadet_peer.h"
 
 #define LOG(level, ...) GNUNET_log_from(level,"cadet-hll",__VA_ARGS__)
 
-
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
-
-
-
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Local peer own ID (memory efficient handle).
- */
-extern GNUNET_PEER_Id myid;
-
-/**
- * Local peer own ID (full value).
- */
-extern struct GNUNET_PeerIdentity my_full_id;
-
-
-/**
- * Don't try to recover tunnels if shutting down.
- */
-extern int shutting_down;
-
-
 /**
  * Hello message of local peer.
  */
-const struct GNUNET_HELLO_Message *mine;
+static struct GNUNET_HELLO_Message *mine;
 
 /**
  * Handle to peerinfo service.
@@ -78,13 +53,9 @@ static struct GNUNET_PEERINFO_Handle *peerinfo;
 /**
  * Iterator context.
  */
-struct GNUNET_PEERINFO_NotifyContext* nc;
+static struct GNUNET_PEERINFO_NotifyContext *nc;
 
 
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
 /**
  * Process each hello message received from peerinfo.
  *
@@ -94,31 +65,37 @@ struct GNUNET_PEERINFO_NotifyContext* nc;
  * @param err_msg Error message.
  */
 static void
-got_hello (void *cls, const struct GNUNET_PeerIdentity *id,
+got_hello (void *cls,
+           const struct GNUNET_PeerIdentity *id,
            const struct GNUNET_HELLO_Message *hello,
            const char *err_msg)
 {
   struct CadetPeer *peer;
 
-  if (NULL == id || NULL == hello)
+  if ( (NULL == id) ||
+       (NULL == hello) )
+    return;
+  if (0 == memcmp (id,
+                   &my_full_id,
+                   sizeof (struct GNUNET_PeerIdentity)))
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " hello with id %p and msg %p\n", id, hello);
+    GNUNET_free_non_null (mine);
+    mine = (struct GNUNET_HELLO_Message *) GNUNET_copy_message (&hello->header);
+    GCD_hello_update ();
     return;
   }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " hello for %s (%d bytes), expires on %s\n",
-       GNUNET_i2s (id), GNUNET_HELLO_size (hello),
-       GNUNET_STRINGS_absolute_time_to_string (GNUNET_HELLO_get_last_expiration(hello)));
-  peer = GCP_get (id, GNUNET_YES);
-  GCP_set_hello (peer, hello);
-
-  if (GCP_get_short_id (peer) == myid)
-    mine = GCP_get_hello (peer);
-}
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Hello for %s (%d bytes), expires on %s\n",
+       GNUNET_i2s (id),
+       GNUNET_HELLO_size (hello),
+       GNUNET_STRINGS_absolute_time_to_string (GNUNET_HELLO_get_last_expiration (hello)));
+  peer = GCP_get (id,
+                  GNUNET_YES);
+  GCP_set_hello (peer,
+                 hello);
+}
 
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
 
 /**
  * Initialize the hello subsystem.
@@ -128,10 +105,12 @@ got_hello (void *cls, const struct GNUNET_PeerIdentity *id,
 void
 GCH_init (const struct GNUNET_CONFIGURATION_Handle *c)
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "init\n");
   GNUNET_assert (NULL == nc);
   peerinfo = GNUNET_PEERINFO_connect (c);
-  nc = GNUNET_PEERINFO_notify (c, GNUNET_NO, &got_hello, NULL);
+  nc = GNUNET_PEERINFO_notify (c,
+                               GNUNET_NO,
+                               &got_hello,
+                               NULL);
 }
 
 
@@ -141,7 +120,6 @@ GCH_init (const struct GNUNET_CONFIGURATION_Handle *c)
 void
 GCH_shutdown ()
 {
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Shutting down channels\n");
   if (NULL != nc)
   {
     GNUNET_PEERINFO_notify_cancel (nc);
@@ -152,6 +130,11 @@ GCH_shutdown ()
     GNUNET_PEERINFO_disconnect (peerinfo);
     peerinfo = NULL;
   }
+  if (NULL != mine)
+  {
+    GNUNET_free (mine);
+    mine = NULL;
+  }
 }
 
 
@@ -166,35 +149,4 @@ GCH_get_mine (void)
   return mine;
 }
 
-
-/**
- * Get another peer's hello message.
- *
- * @param id ID of the peer whose hello message is requested.
- *
- * @return Hello message, if any (NULL possible).
- */
-const struct GNUNET_HELLO_Message *
-GCH_get (const struct GNUNET_PeerIdentity *id)
-{
-  struct CadetPeer *p;
-
-  p = GCP_get (id, GNUNET_NO);
-  if (NULL == p)
-    return NULL;
-  return GCP_get_hello (p);
-}
-
-
-/**
- * Convert a hello message to a string.
- *
- * @param h Hello message.
- */
-char *
-GCH_2s (const struct GNUNET_HELLO_Message *h)
-{
-  return "hello (TODO)";
-}
-
-
+/* end of gnunet-service-cadet-new_hello.c */
diff --git a/src/cadet/gnunet-service-cadet_hello.h b/src/cadet/gnunet-service-cadet_hello.h
index 34121e1e0..4291ae985 100644
--- a/src/cadet/gnunet-service-cadet_hello.h
+++ b/src/cadet/gnunet-service-cadet_hello.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2014 GNUnet e.V.
+     Copyright (C) 2014, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -22,8 +22,9 @@
  * @file cadet/gnunet-service-cadet_hello.h
  * @brief cadet service; dealing with hello messages
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
  *
- * All functions in this file should use the prefix GMH (Gnunet Cadet Hello)
+ * All functions in this file should use the prefix GCH (Gnunet Cadet Hello)
  */
 
 #ifndef GNUNET_SERVICE_CADET_HELLO_H
diff --git a/src/cadet/gnunet-service-cadet_local.c b/src/cadet/gnunet-service-cadet_local.c
deleted file mode 100644
index dea6681df..000000000
--- a/src/cadet/gnunet-service-cadet_local.c
+++ /dev/null
@@ -1,1553 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#include "gnunet_statistics_service.h"
-
-#include "cadet.h"
-#include "cadet_protocol.h" /* GNUNET_CADET_Data is shared */
-
-#include "gnunet-service-cadet_local.h"
-#include "gnunet-service-cadet_channel.h"
-
-/* INFO DEBUG */
-#include "gnunet-service-cadet_tunnel.h"
-#include "gnunet-service-cadet_peer.h"
-
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-loc",__VA_ARGS__)
-
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
-
-/**
- * Struct containing information about a client of the service
- *
- * TODO: add a list of 'waiting' ports
- */
-struct CadetClient
-{
-  /**
-   * Linked list next
-   */
-  struct CadetClient *next;
-
-  /**
-   * Linked list prev
-   */
-  struct CadetClient *prev;
-
-  /**
-   * Tunnels that belong to this client, indexed by local id
-   */
-  struct GNUNET_CONTAINER_MultiHashMap32 *own_channels;
-
-  /**
-   * Tunnels this client has accepted, indexed by incoming local id
-   */
-  struct GNUNET_CONTAINER_MultiHashMap32 *incoming_channels;
-
-  /**
-   * Channel ID for the next incoming channel.
-   */
-  struct GNUNET_CADET_ClientChannelNumber next_ccn;
-
-  /**
-   * Handle to communicate with the client
-   */
-  struct GNUNET_SERVER_Client *handle;
-
-  /**
-   * Ports that this client has declared interest in.
-   * Indexed by port, contains *Client.
-   */
-  struct GNUNET_CONTAINER_MultiHashMap *ports;
-
-  /**
-   * Whether the client is active or shutting down (don't send confirmations
-   * to a client that is shutting down.
-   */
-  int shutting_down;
-
-  /**
-   * ID of the client, mainly for debug messages
-   */
-  unsigned int id;
-};
-
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Handle to server lib.
- */
-static struct GNUNET_SERVER_Handle *server_handle;
-
-/**
- * DLL with all the clients, head.
- */
-static struct CadetClient *clients_head;
-
-/**
- * DLL with all the clients, tail.
- */
-static struct CadetClient *clients_tail;
-
-/**
- * Next ID to assign to a client.
- */
-unsigned int next_client_id;
-
-/**
- * All ports clients of this peer have opened.
- */
-static struct GNUNET_CONTAINER_MultiHashMap *ports;
-
-/**
- * Notification context, to send messages to local clients.
- */
-static struct GNUNET_SERVER_NotificationContext *nc;
-
-
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
-/**
- * Remove client's ports from the global hashmap on disconnect.
- *
- * @param cls Closure (unused).
- * @param key Port.
- * @param value Client structure.
- *
- * @return #GNUNET_OK, keep iterating.
- */
-static int
-client_release_ports (void *cls,
-                      const struct GNUNET_HashCode *key,
-                      void *value)
-{
-  int res;
-
-  res = GNUNET_CONTAINER_multihashmap_remove (ports, key, value);
-  if (GNUNET_YES != res)
-  {
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Port %s by client %p was not registered.\n",
-         GNUNET_h2s (key), value);
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Iterator for deleting each channel whose client endpoint disconnected.
- *
- * @param cls Closure (client that has disconnected).
- * @param key The local channel id (used to access the hashmap).
- * @param value The value stored at the key (channel to destroy).
- *
- * @return #GNUNET_OK, keep iterating.
- */
-static int
-channel_destroy_iterator (void *cls,
-                          uint32_t key,
-                          void *value)
-{
-  struct CadetChannel *ch = value;
-  struct CadetClient *c = cls;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " Channel %s destroy, due to client %s shutdown.\n",
-       GCCH_2s (ch), GML_2s (c));
-
-  GCCH_handle_local_destroy (ch,
-                             c,
-                             key < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-  return GNUNET_OK;
-}
-
-
-/**
- * Unregister data and free memory for a client.
- *
- * @param c Client to destroy. No longer valid after call.
- */
-static void
-client_destroy (struct CadetClient *c)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  client destroy: %p/%u\n", c, c->id);
-  GNUNET_SERVER_client_drop (c->handle);
-  c->shutting_down = GNUNET_YES;
-
-  if (NULL != c->own_channels)
-  {
-    GNUNET_CONTAINER_multihashmap32_iterate (c->own_channels,
-                                             &channel_destroy_iterator, c);
-    GNUNET_CONTAINER_multihashmap32_destroy (c->own_channels);
-  }
-  if (NULL != c->incoming_channels)
-  {
-    GNUNET_CONTAINER_multihashmap32_iterate (c->incoming_channels,
-                                             &channel_destroy_iterator, c);
-    GNUNET_CONTAINER_multihashmap32_destroy (c->incoming_channels);
-  }
-  if (NULL != c->ports)
-  {
-    GNUNET_CONTAINER_multihashmap_iterate (c->ports,
-                                           &client_release_ports, c);
-    GNUNET_CONTAINER_multihashmap_destroy (c->ports);
-  }
-
-  GNUNET_CONTAINER_DLL_remove (clients_head, clients_tail, c);
-  GNUNET_STATISTICS_update (stats, "# clients", -1, GNUNET_NO);
-  GNUNET_SERVER_client_set_user_context (c->handle, NULL);
-  GNUNET_free (c);
-}
-
-
-/**
- * Create a client record, register data and initialize memory.
- *
- * @param client Client's handle.
- */
-static struct CadetClient *
-client_new (struct GNUNET_SERVER_Client *client)
-{
-  struct CadetClient *c;
-
-  GNUNET_SERVER_client_keep (client);
-  GNUNET_SERVER_notification_context_add (nc, client);
-
-  c = GNUNET_new (struct CadetClient);
-  c->handle = client;
-  c->id = next_client_id++; /* overflow not important: just for debug */
-
-  c->own_channels = GNUNET_CONTAINER_multihashmap32_create (32);
-  c->incoming_channels = GNUNET_CONTAINER_multihashmap32_create (32);
-
-  GNUNET_SERVER_client_set_user_context (client, c);
-  GNUNET_CONTAINER_DLL_insert (clients_head, clients_tail, c);
-  GNUNET_STATISTICS_update (stats, "# clients", +1, GNUNET_NO);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  client created: %p/%u\n", c, c->id);
-
-  return c;
-}
-
-
-/******************************************************************************/
-/********************************  HANDLES  ***********************************/
-/******************************************************************************/
-
-/**
- * Handler for client connection.
- *
- * @param cls Closure (unused).
- * @param client Client handler.
- */
-static void
-handle_client_connect (void *cls, struct GNUNET_SERVER_Client *client)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Client connected: %p\n", client);
-  if (NULL == client)
-    return;
-
-  (void) client_new (client);
-}
-
-
-/**
- * Handler for client disconnection
- *
- * @param cls closure
- * @param client identification of the client; NULL
- *        for the last call when the server is destroyed
- */
-static void
-handle_client_disconnect (void *cls, struct GNUNET_SERVER_Client *client)
-{
-  struct CadetClient *c;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Client disconnected: %p\n", client);
-
-  c = GML_client_get (client);
-  if (NULL != c)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "matching client found (%u, %p)\n",
-                c->id, c);
-    client_destroy (c);
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " disconnecting client's context NULL\n");
-  }
-  return;
-}
-
-
-/**
- * Handler for port open requests.
- *
- * @param cls Closure.
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_port_open (void *cls, struct GNUNET_SERVER_Client *client,
-                  const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-  struct GNUNET_CADET_PortMessage *pmsg;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "open port requested\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  /* Message size sanity check */
-  if (sizeof (struct GNUNET_CADET_PortMessage) != ntohs (message->size))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  pmsg = (struct GNUNET_CADET_PortMessage *) message;
-  if (NULL == c->ports)
-  {
-    c->ports = GNUNET_CONTAINER_multihashmap_create (4, GNUNET_NO);
-  }
-  /* store in client's hashmap */
-  if (GNUNET_OK !=
-      GNUNET_CONTAINER_multihashmap_put (c->ports, &pmsg->port, c,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  /* store in global hashmap */
-  /* FIXME only allow one client to have the port open,
-   *       have a backup hashmap with waiting clients */
-  GNUNET_CONTAINER_multihashmap_put (ports, &pmsg->port, c,
-                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for port close requests.
- *
- * @param cls Closure.
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_port_close (void *cls, struct GNUNET_SERVER_Client *client,
-                   const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-  struct GNUNET_CADET_PortMessage *pmsg;
-  int removed;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "close port requested\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-    /* Message size sanity check */
-  if (sizeof (struct GNUNET_CADET_PortMessage) != ntohs (message->size))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  pmsg = (struct GNUNET_CADET_PortMessage *) message;
-  removed = GNUNET_CONTAINER_multihashmap_remove (c->ports, &pmsg->port, c);
-  GNUNET_break_op (GNUNET_YES == removed);
-  removed = GNUNET_CONTAINER_multihashmap_remove (ports, &pmsg->port, c);
-  GNUNET_break_op (GNUNET_YES == removed);
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for requests of new channels.
- *
- * @param cls Closure.
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_channel_create (void *cls, struct GNUNET_SERVER_Client *client,
-                       const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "\n");
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "new channel requested\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  /* Message size sanity check */
-  if (sizeof (struct GNUNET_CADET_LocalChannelCreateMessage)
-      != ntohs (message->size))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  if (GNUNET_OK !=
-      GCCH_handle_local_create (c,
-                                (struct GNUNET_CADET_LocalChannelCreateMessage *)
-                                message))
-  {
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for requests of deleting tunnels
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- */
-static void
-handle_channel_destroy (void *cls, struct GNUNET_SERVER_Client *client,
-                        const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_CADET_LocalChannelDestroyMessage *msg;
-  struct CadetClient *c;
-  struct CadetChannel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Got a DESTROY CHANNEL from client!\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  /* Message sanity check */
-  if (sizeof (struct GNUNET_CADET_LocalChannelDestroyMessage)
-      != ntohs (message->size))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  msg = (const struct GNUNET_CADET_LocalChannelDestroyMessage *) message;
-
-  /* Retrieve tunnel */
-  ccn = msg->ccn;
-  ch = GML_channel_get (c, ccn);
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "Client %u is destroying channel %X\n",
-       c->id, ccn);
-
-  if (NULL == ch)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING, "  channel %X not found\n", ccn);
-    GNUNET_STATISTICS_update (stats,
-                              "# client destroy messages on unknown channel",
-                              1, GNUNET_NO);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
-  }
-
-  GCCH_handle_local_destroy (ch,
-                             c,
-                             ntohl (ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for client traffic
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- */
-static void
-handle_data (void *cls, struct GNUNET_SERVER_Client *client,
-             const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_MessageHeader *payload;
-  struct GNUNET_CADET_LocalData *msg;
-  struct CadetClient *c;
-  struct CadetChannel *ch;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-  size_t message_size;
-  size_t payload_size;
-  size_t payload_claimed_size;
-  int fwd;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "\n");
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "\n");
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Got data from a client\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  /* Sanity check for message size */
-  message_size = ntohs (message->size);
-  if (sizeof (struct GNUNET_CADET_LocalData)
-      + sizeof (struct GNUNET_MessageHeader) > message_size
-      || GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE < message_size)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  /* Sanity check for payload size */
-  payload_size = message_size - sizeof (struct GNUNET_CADET_LocalData);
-  msg = (struct GNUNET_CADET_LocalData *) message;
-  payload = (struct GNUNET_MessageHeader *) &msg[1];
-  payload_claimed_size = ntohs (payload->size);
-  if (sizeof (struct GNUNET_MessageHeader) > payload_claimed_size
-      || GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE < payload_claimed_size
-      || payload_claimed_size > payload_size)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "client claims to send %u bytes in %u payload\n",
-         payload_claimed_size, payload_size);
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  ccn = msg->ccn;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  %u bytes (%u payload) by client %u\n",
-       payload_size, payload_claimed_size, c->id);
-
-  /* Channel exists? */
-  fwd = ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
-  ch = GML_channel_get (c, ccn);
-  if (NULL == ch)
-  {
-    GNUNET_STATISTICS_update (stats,
-                              "# client data messages on unknown channel",
-                              1, GNUNET_NO);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
-  }
-
-  if (GNUNET_OK != GCCH_handle_local_data (ch, c, fwd, payload, payload_size))
-  {
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "receive done OK\n");
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-
-  return;
-}
-
-
-/**
- * Handler for client's ACKs for payload traffic.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_ack (void *cls, struct GNUNET_SERVER_Client *client,
-            const struct GNUNET_MessageHeader *message)
-{
-  struct GNUNET_CADET_LocalAck *msg;
-  struct CadetChannel *ch;
-  struct CadetClient *c;
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-  int fwd;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "\n");
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Got a local ACK\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  msg = (struct GNUNET_CADET_LocalAck *) message;
-
-  /* Channel exists? */
-  ccn = msg->ccn;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  on channel %X\n",
-       ntohl (ccn.channel_of_client));
-  ch = GML_channel_get (c, ccn);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "   -- ch %p\n", ch);
-  if (NULL == ch)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Channel %X unknown.\n",
-         ntohl (ccn.channel_of_client));
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  for client %u.\n", c->id);
-    GNUNET_STATISTICS_update (stats,
-                              "# client ack messages on unknown channel",
-                              1, GNUNET_NO);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
-  }
-
-  /* If client is root, the ACK is going FWD, therefore this is "BCK ACK". */
-  /* If client is dest, the ACK is going BCK, therefore this is "FWD ACK" */
-  fwd = ntohl (ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
-
-  GCCH_handle_local_ack (ch, fwd);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Iterator over all peers to send a monitoring client info about each peer.
- *
- * @param cls Closure ().
- * @param peer Peer ID (tunnel remote peer).
- * @param value Peer info.
- *
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-get_all_peers_iterator (void *cls,
-                        const struct GNUNET_PeerIdentity * peer,
-                        void *value)
-{
-  struct GNUNET_SERVER_Client *client = cls;
-  struct CadetPeer *p = value;
-  struct GNUNET_CADET_LocalInfoPeer msg;
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
-  msg.destination = *peer;
-  msg.paths = htons (GCP_count_paths (p));
-  msg.tunnel = htons (NULL != GCP_get_tunnel (p));
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "sending info about peer %s\n",
-       GNUNET_i2s (peer));
-
-  GNUNET_SERVER_notification_context_unicast (nc, client,
-                                              &msg.header, GNUNET_NO);
-  return GNUNET_YES;
-}
-
-
-/**
- * Iterator over all peers to dump info for each peer.
- *
- * @param cls Closure (unused).
- * @param peer Peer ID (tunnel remote peer).
- * @param value Peer info.
- *
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-show_peer_iterator (void *cls,
-                        const struct GNUNET_PeerIdentity * peer,
-                        void *value)
-{
-  struct CadetPeer *p = value;
-  struct CadetTunnel *t;
-
-  t = GCP_get_tunnel (p);
-  if (NULL != t)
-    GCT_debug (t, GNUNET_ERROR_TYPE_ERROR);
-
-  LOG (GNUNET_ERROR_TYPE_ERROR, "\n");
-
-  return GNUNET_YES;
-}
-
-
-/**
- * Iterator over all paths of a peer to build an InfoPeer message.
- *
- * Message contains blocks of peers, first not included.
- *
- * @param cls Closure (message to build).
- * @param peer Peer this path is towards.
- * @param path Path itself
- * @return #GNUNET_YES if should keep iterating.
- *         #GNUNET_NO otherwise.
- */
-static int
-path_info_iterator (void *cls,
-                    struct CadetPeer *peer,
-                    struct CadetPeerPath *path)
-{
-  struct GNUNET_CADET_LocalInfoPeer *resp = cls;
-  struct GNUNET_PeerIdentity *id;
-  uint16_t msg_size;
-  uint16_t path_size;
-  unsigned int i;
-
-  msg_size = ntohs (resp->header.size);
-  path_size = sizeof (struct GNUNET_PeerIdentity) * (path->length - 1);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Info Path %u\n", path->length);
-  if (msg_size + path_size > UINT16_MAX)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING, "path too long for info message\n");
-    return GNUNET_NO;
-  }
-
-  i = msg_size - sizeof (struct GNUNET_CADET_LocalInfoPeer);
-  i = i / sizeof (struct GNUNET_PeerIdentity);
-
-  /* Set id to the address of the first free peer slot. */
-  id = (struct GNUNET_PeerIdentity *) &resp[1];
-  id = &id[i];
-
-  /* Don't copy first peers.
-   * First peer is always the local one.
-   * Last peer is always the destination (leave as 0, EOL).
-   */
-  for (i = 0; i < path->length - 1; i++)
-  {
-    GNUNET_PEER_resolve (path->peers[i + 1], &id[i]);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " %s\n", GNUNET_i2s (&id[i]));
-  }
-
-  resp->header.size = htons (msg_size + path_size);
-
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's INFO PEERS request.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_get_peers (void *cls, struct GNUNET_SERVER_Client *client,
-                    const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-  struct GNUNET_MessageHeader reply;
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received get peers request from client %u (%p)\n",
-       c->id, client);
-
-  GCP_iterate_all (get_all_peers_iterator, client);
-  reply.size = htons (sizeof (reply));
-  reply.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS);
-  GNUNET_SERVER_notification_context_unicast (nc, client, &reply, GNUNET_NO);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Get peers request from client %u completed\n", c->id);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for client's SHOW_PEER request.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-void
-handle_show_peer (void *cls, struct GNUNET_SERVER_Client *client,
-                  const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_CADET_LocalInfo *msg;
-  struct GNUNET_CADET_LocalInfoPeer *resp;
-  struct CadetPeer *p;
-  struct CadetClient *c;
-  unsigned char cbuf[64 * 1024];
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  msg = (struct GNUNET_CADET_LocalInfo *) message;
-  resp = (struct GNUNET_CADET_LocalInfoPeer *) cbuf;
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "Received peer info request from client %u for peer %s\n",
-       c->id, GNUNET_i2s_full (&msg->peer));
-
-  resp->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER);
-  resp->header.size = htons (sizeof (struct GNUNET_CADET_LocalInfoPeer));
-  resp->destination = msg->peer;
-  p = GCP_get (&msg->peer, GNUNET_NO);
-  if (NULL == p)
-  {
-    /* We don't know the peer */
-
-    LOG (GNUNET_ERROR_TYPE_INFO, "Peer %s unknown\n",
-         GNUNET_i2s_full (&msg->peer));
-    resp->paths = htons (0);
-    resp->tunnel = htons (NULL != GCP_get_tunnel (p));
-
-    GNUNET_SERVER_notification_context_unicast (nc, client,
-                                                &resp->header,
-                                                GNUNET_NO);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
-  }
-
-  resp->paths = htons (GCP_count_paths (p));
-  resp->tunnel = htons (NULL != GCP_get_tunnel (p));
-  GCP_iterate_paths (p, &path_info_iterator, resp);
-
-  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
-                                              &resp->header, GNUNET_NO);
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "Show peer from client %u completed.\n", c->id);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Iterator over all tunnels to send a monitoring client info about each tunnel.
- *
- * @param cls Closure ().
- * @param peer Peer ID (tunnel remote peer).
- * @param value Tunnel info.
- *
- * @return #GNUNET_YES, to keep iterating.
- */
-static int
-get_all_tunnels_iterator (void *cls,
-                          const struct GNUNET_PeerIdentity * peer,
-                          void *value)
-{
-  struct GNUNET_SERVER_Client *client = cls;
-  struct CadetTunnel *t = value;
-  struct GNUNET_CADET_LocalInfoTunnel msg;
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
-  msg.destination = *peer;
-  msg.channels = htonl (GCT_count_channels (t));
-  msg.connections = htonl (GCT_count_any_connections (t));
-  msg.cstate = htons ((uint16_t) GCT_get_cstate (t));
-  msg.estate = htons ((uint16_t) GCT_get_estate (t));
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "sending info about tunnel ->%s\n",
-       GNUNET_i2s (peer));
-
-  GNUNET_SERVER_notification_context_unicast (nc, client,
-                                              &msg.header, GNUNET_NO);
-  return GNUNET_YES;
-}
-
-
-/**
- * Handler for client's INFO TUNNELS request.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_get_tunnels (void *cls, struct GNUNET_SERVER_Client *client,
-                    const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-  struct GNUNET_MessageHeader reply;
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received get tunnels request from client %u (%p)\n",
-       c->id, client);
-
-  GCT_iterate_all (get_all_tunnels_iterator, client);
-  reply.size = htons (sizeof (reply));
-  reply.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS);
-  GNUNET_SERVER_notification_context_unicast (nc, client, &reply, GNUNET_NO);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Get tunnels request from client %u completed\n", c->id);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-static void
-iter_connection (void *cls, struct CadetConnection *c)
-{
-  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
-  struct GNUNET_CADET_ConnectionTunnelIdentifier *h;
-
-  h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
-  h[msg->connections] = *(GCC_get_id (c));
-  msg->connections++;
-}
-
-static void
-iter_channel (void *cls, struct CadetChannel *ch)
-{
-  struct GNUNET_CADET_LocalInfoTunnel *msg = cls;
-  struct GNUNET_CADET_ConnectionTunnelIdentifier *h = (struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
-  struct GNUNET_CADET_ChannelTunnelNumber *chn = (struct GNUNET_CADET_ChannelTunnelNumber *) &h[msg->connections];
-
-  chn[msg->channels] = GCCH_get_id (ch);
-  msg->channels++;
-}
-
-
-/**
- * Handler for client's SHOW_TUNNEL request.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-void
-handle_show_tunnel (void *cls, struct GNUNET_SERVER_Client *client,
-                    const struct GNUNET_MessageHeader *message)
-{
-  const struct GNUNET_CADET_LocalInfo *msg;
-  struct GNUNET_CADET_LocalInfoTunnel *resp;
-  struct CadetClient *c;
-  struct CadetTunnel *t;
-  unsigned int ch_n;
-  unsigned int c_n;
-  size_t size;
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  msg = (struct GNUNET_CADET_LocalInfo *) message;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received tunnel info request from client %u for tunnel %s\n",
-       c->id, GNUNET_i2s_full(&msg->peer));
-
-  t = GCP_get_tunnel (GCP_get (&msg->peer, GNUNET_NO));
-  if (NULL == t)
-  {
-    /* We don't know the tunnel */
-    struct GNUNET_CADET_LocalInfoTunnel warn;
-
-    LOG (GNUNET_ERROR_TYPE_INFO, "Tunnel %s unknown %u\n",
-         GNUNET_i2s_full(&msg->peer), sizeof (warn));
-    warn.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
-    warn.header.size = htons (sizeof (warn));
-    warn.destination = msg->peer;
-    warn.channels = htonl (0);
-    warn.connections = htonl (0);
-    warn.cstate = htons (0);
-    warn.estate = htons (0);
-
-    GNUNET_SERVER_notification_context_unicast (nc, client,
-                                                &warn.header,
-                                                GNUNET_NO);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
-  }
-
-  /* Initialize context */
-  ch_n = GCT_count_channels (t);
-  c_n = GCT_count_any_connections (t);
-
-  size = sizeof (struct GNUNET_CADET_LocalInfoTunnel);
-  size += c_n * sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier);
-  size += ch_n * sizeof (struct GNUNET_CADET_ChannelTunnelNumber);
-
-  resp = GNUNET_malloc (size);
-  resp->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL);
-  resp->header.size = htons (size);
-  resp->destination = msg->peer;
-  /* Do not interleave with iterators, iter_channel needs conn in HBO */
-  GCT_iterate_connections (t, &iter_connection, resp);
-  GCT_iterate_channels (t, &iter_channel, resp);
-  resp->connections = htonl (resp->connections);
-  resp->channels = htonl (resp->channels);
-  /* Do not interleave end */
-  resp->cstate = htons (GCT_get_cstate (t));
-  resp->estate = htons (GCT_get_estate (t));
-  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
-                                              &resp->header, GNUNET_NO);
-  GNUNET_free (resp);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Show tunnel request from client %u completed. %u conn, %u ch\n",
-       c->id, c_n, ch_n);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for client's INFO_DUMP request.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-void
-handle_info_dump (void *cls, struct GNUNET_SERVER_Client *client,
-                  const struct GNUNET_MessageHeader *message)
-{
-  struct CadetClient *c;
-
-  /* Sanity check for client registration */
-  if (NULL == (c = GML_client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "Received dump info request from client %u\n",
-       c->id);
-  LOG (GNUNET_ERROR_TYPE_ERROR,
-       "*************************** DUMP START ***************************\n");
-
-  for (c = clients_head; NULL != c; c = c->next)
-  {
-    LOG (GNUNET_ERROR_TYPE_ERROR, "Client %u (%p), handle: %p\n",
-         c->id, c, c->handle);
-    if (NULL != c->ports)
-      LOG (GNUNET_ERROR_TYPE_ERROR, "\t%3u ports registered\n",
-           GNUNET_CONTAINER_multihashmap_size (c->ports));
-    else
-      LOG (GNUNET_ERROR_TYPE_ERROR, "\t no ports registered\n");
-    LOG (GNUNET_ERROR_TYPE_ERROR, "\t%3u own channles\n",
-         GNUNET_CONTAINER_multihashmap32_size (c->own_channels));
-    LOG (GNUNET_ERROR_TYPE_ERROR, "\t%3u incoming channles\n",
-         GNUNET_CONTAINER_multihashmap32_size (c->incoming_channels));
-  }
-  LOG (GNUNET_ERROR_TYPE_ERROR, "***************************\n");
-  GCP_iterate_all (&show_peer_iterator, NULL);
-
-  LOG (GNUNET_ERROR_TYPE_ERROR,
-       "**************************** DUMP END ****************************\n");
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Functions to handle messages from clients
- */
-static struct GNUNET_SERVER_MessageHandler client_handlers[] = {
-  {&handle_port_open, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN,
-    sizeof (struct GNUNET_CADET_PortMessage)},
-  {&handle_port_close, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE,
-    sizeof (struct GNUNET_CADET_PortMessage)},
-  {&handle_channel_create, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE,
-   sizeof (struct GNUNET_CADET_LocalChannelCreateMessage)},
-  {&handle_channel_destroy, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY,
-   sizeof (struct GNUNET_CADET_LocalChannelDestroyMessage)},
-  {&handle_data, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA, 0},
-  {&handle_ack, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
-   sizeof (struct GNUNET_CADET_LocalAck)},
-  {&handle_get_peers, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
-   sizeof (struct GNUNET_MessageHeader)},
-  {&handle_show_peer, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
-   sizeof (struct GNUNET_CADET_LocalInfo)},
-  {&handle_get_tunnels, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
-   sizeof (struct GNUNET_MessageHeader)},
-  {&handle_show_tunnel, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
-   sizeof (struct GNUNET_CADET_LocalInfo)},
-  {&handle_info_dump, NULL, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_DUMP,
-   sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Initialize server subsystem.
- *
- * @param handle Server handle.
- */
-void
-GML_init (struct GNUNET_SERVER_Handle *handle)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "init\n");
-  server_handle = handle;
-  GNUNET_SERVER_suspend (server_handle);
-  ports = GNUNET_CONTAINER_multihashmap_create (16, GNUNET_NO);
-}
-
-
-/**
- * Install server (service) handlers and start listening to clients.
- */
-void
-GML_start (void)
-{
-  GNUNET_SERVER_add_handlers (server_handle, client_handlers);
-  GNUNET_SERVER_connect_notify (server_handle,  &handle_client_connect, NULL);
-  GNUNET_SERVER_disconnect_notify (server_handle, &handle_client_disconnect,
-                                   NULL);
-  nc = GNUNET_SERVER_notification_context_create (server_handle, 1);
-
-  clients_head = NULL;
-  clients_tail = NULL;
-  next_client_id = 0;
-  GNUNET_SERVER_resume (server_handle);
-}
-
-
-/**
- * Shutdown server.
- */
-void
-GML_shutdown (void)
-{
-  struct CadetClient *c;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Shutting down local\n");
-
-  for (c = clients_head; NULL != clients_head; c = clients_head)
-    client_destroy (c);
-
-  if (nc != NULL)
-  {
-    GNUNET_SERVER_notification_context_destroy (nc);
-    nc = NULL;
-  }
-
-}
-
-
-/**
- * Get a channel from a client.
- *
- * @param c Client to check.
- * @param ccn Channel ID, must be local (> 0x800...).
- *
- * @return non-NULL if channel exists in the clients lists
- */
-struct CadetChannel *
-GML_channel_get (struct CadetClient *c,
-                 struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  struct GNUNET_CONTAINER_MultiHashMap32 *map;
-
-  if (ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-    map = c->own_channels;
-  else
-    map = c->incoming_channels;
-
-  if (NULL == map)
-  {
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Client %s does no t have a valid map for CCN %X\n",
-         GML_2s (c), ccn);
-    return NULL;
-  }
-  return GNUNET_CONTAINER_multihashmap32_get (map,
-                                              ccn.channel_of_client);
-}
-
-
-/**
- * Add a channel to a client
- *
- * @param client Client.
- * @param ccn Channel ID.
- * @param ch Channel.
- */
-void
-GML_channel_add (struct CadetClient *client,
-                 struct GNUNET_CADET_ClientChannelNumber ccn,
-                 struct CadetChannel *ch)
-{
-  if (ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-    GNUNET_CONTAINER_multihashmap32_put (client->own_channels,
-                                         ccn.channel_of_client,
-                                         ch,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
-  else
-    GNUNET_CONTAINER_multihashmap32_put (client->incoming_channels,
-                                         ccn.channel_of_client,
-                                         ch,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
-}
-
-
-/**
- * Remove a channel from a client.
- *
- * @param client Client.
- * @param ccn Channel ID.
- * @param ch Channel.
- */
-void
-GML_channel_remove (struct CadetClient *client,
-                    struct GNUNET_CADET_ClientChannelNumber ccn,
-                    struct CadetChannel *ch)
-{
-  if (ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-    GNUNET_CONTAINER_multihashmap32_remove (client->own_channels,
-                                            ccn.channel_of_client,
-                                            ch);
-  else
-    GNUNET_CONTAINER_multihashmap32_remove (client->incoming_channels,
-                                            ccn.channel_of_client,
-                                            ch);
-}
-
-
-/**
- * Get the tunnel's next free local channel ID.
- *
- * @param c Client.
- *
- * @return LID of a channel free to use.
- */
-struct GNUNET_CADET_ClientChannelNumber
-GML_get_next_ccn (struct CadetClient *c)
-{
-  struct GNUNET_CADET_ClientChannelNumber ccn;
-
-  while (NULL != GML_channel_get (c,
-                                  c->next_ccn))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Channel %u exists...\n",
-         c->next_ccn);
-    c->next_ccn.channel_of_client
-      = htonl (1 + (ntohl (c->next_ccn.channel_of_client)));
-    if (ntohl (c->next_ccn.channel_of_client) >=
-        GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-      c->next_ccn.channel_of_client = htonl (0);
-  }
-  ccn = c->next_ccn;
-  c->next_ccn.channel_of_client
-    = htonl (1 + (ntohl (c->next_ccn.channel_of_client)));
-
-  return ccn;
-}
-
-
-/**
- * Check if client has registered with the service and has not disconnected
- *
- * @param client the client to check
- *
- * @return non-NULL if client exists in the global DLL
- */
-struct CadetClient *
-GML_client_get (struct GNUNET_SERVER_Client *client)
-{
-  if (NULL == client)
-    return NULL;
-  return GNUNET_SERVER_client_get_user_context (client,
-                                                struct CadetClient);
-}
-
-
-/**
- * Find a client that has opened a port
- *
- * @param port Port to check.
- *
- * @return non-NULL if a client has the port.
- */
-struct CadetClient *
-GML_client_get_by_port (const struct GNUNET_HashCode *port)
-{
-  return GNUNET_CONTAINER_multihashmap_get (ports, port);
-}
-
-
-/**
- * Deletes a channel from a client (either owner or destination).
- *
- * @param c Client whose tunnel to delete.
- * @param ch Channel which should be deleted.
- * @param id Channel ID.
- */
-void
-GML_client_delete_channel (struct CadetClient *c,
-                           struct CadetChannel *ch,
-                           struct GNUNET_CADET_ClientChannelNumber id)
-{
-  int res;
-
-  if (ntohl (id.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
-  {
-    res = GNUNET_CONTAINER_multihashmap32_remove (c->own_channels,
-                                                  id.channel_of_client,
-                                                  ch);
-    if (GNUNET_YES != res)
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "client_delete_tunnel root KO\n");
-  }
-  else
-  {
-    res = GNUNET_CONTAINER_multihashmap32_remove (c->incoming_channels,
-                                                  id.channel_of_client,
-                                                  ch);
-    if (GNUNET_YES != res)
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "client_delete_channel dest KO\n");
-  }
-}
-
-/**
- * Build a local ACK message and send it to a local client, if needed.
- *
- * If the client was already allowed to send data, do nothing.
- *
- * @param c Client to whom send the ACK.
- * @param ccn Channel ID to use
- */
-void
-GML_send_ack (struct CadetClient *c,
-              struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  struct GNUNET_CADET_LocalAck msg;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "send local %s ack on %X towards %p\n",
-       ntohl (ccn.channel_of_client) < GNUNET_CADET_LOCAL_CHANNEL_ID_CLI
-       ? "FWD" : "BCK",
-       ntohl (ccn.channel_of_client),
-       c);
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
-  msg.ccn = ccn;
-  GNUNET_SERVER_notification_context_unicast (nc,
-                                              c->handle,
-                                              &msg.header,
-                                              GNUNET_NO);
-
-}
-
-
-
-/**
- * Notify the client that a new incoming channel was created.
- *
- * @param c Client to notify.
- * @param ccn Channel ID.
- * @param port Channel's destination port.
- * @param opt Options (bit array).
- * @param peer Origin peer.
- */
-void
-GML_send_channel_create (struct CadetClient *c,
-                         struct GNUNET_CADET_ClientChannelNumber ccn,
-                         const struct GNUNET_HashCode *port,
-                         uint32_t opt,
-                         const struct GNUNET_PeerIdentity *peer)
-{
-  struct GNUNET_CADET_LocalChannelCreateMessage msg;
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
-  msg.ccn = ccn;
-  msg.port = *port;
-  msg.opt = htonl (opt);
-  msg.peer = *peer;
-  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
-                                              &msg.header, GNUNET_NO);
-}
-
-
-/**
- * Build a local channel NACK message and send it to a local client.
- *
- * @param c Client to whom send the NACK.
- * @param ccn Channel ID to use
- */
-void
-GML_send_channel_nack (struct CadetClient *c,
-                       struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  struct GNUNET_CADET_LocalAck msg;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "send local nack on %X towards %p\n",
-       ntohl (ccn.channel_of_client),
-       c);
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED);
-  msg.ccn = ccn;
-  GNUNET_SERVER_notification_context_unicast (nc,
-                                              c->handle,
-                                              &msg.header,
-                                              GNUNET_NO);
-
-}
-
-/**
- * Notify a client that a channel is no longer valid.
- *
- * @param c Client.
- * @param ccn ID of the channel that is destroyed.
- */
-void
-GML_send_channel_destroy (struct CadetClient *c,
-                          struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  struct GNUNET_CADET_LocalChannelDestroyMessage msg;
-
-  if (NULL == c)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  if (GNUNET_YES == c->shutting_down)
-    return;
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
-  msg.ccn = ccn;
-  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
-                                              &msg.header, GNUNET_NO);
-}
-
-
-/**
- * Modify the cadet message ID from global to local and send to client.
- *
- * @param c Client to send to.
- * @param msg Message to modify and send.
- * @param ccn Channel ID to use (c can be both owner and client).
- */
-void
-GML_send_data (struct CadetClient *c,
-               const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-               struct GNUNET_CADET_ClientChannelNumber ccn)
-{
-  struct GNUNET_CADET_LocalData *copy;
-  uint16_t size = ntohs (msg->header.size) - sizeof (struct GNUNET_CADET_ChannelAppDataMessage);
-  char cbuf[size + sizeof (struct GNUNET_CADET_LocalData)];
-
-  if (size < sizeof (struct GNUNET_MessageHeader))
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-  if (NULL == c)
-  {
-    GNUNET_break (0);
-    return;
-  }
-  copy = (struct GNUNET_CADET_LocalData *) cbuf;
-  GNUNET_memcpy (&copy[1], &msg[1], size);
-  copy->header.size = htons (sizeof (struct GNUNET_CADET_LocalData) + size);
-  copy->header.type = htons (GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
-  copy->ccn = ccn;
-  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
-                                              &copy->header, GNUNET_NO);
-}
-
-
-/**
- * Get the static string to represent a client.
- *
- * @param c Client.
- *
- * @return Static string for the client.
- */
-const char *
-GML_2s (const struct CadetClient *c)
-{
-  static char buf[32];
-
-  SPRINTF (buf, "%u", c->id);
-  return buf;
-}
diff --git a/src/cadet/gnunet-service-cadet_local.h b/src/cadet/gnunet-service-cadet_local.h
deleted file mode 100644
index 113c2f489..000000000
--- a/src/cadet/gnunet-service-cadet_local.h
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet_local.h
- * @brief cadet service; dealing with local clients
- * @author Bartlomiej Polot
- *
- * All functions in this file should use the prefix GML (Gnunet Cadet Local)
- */
-
-#ifndef GNUNET_SERVICE_CADET_LOCAL_H
-#define GNUNET_SERVICE_CADET_LOCAL_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-/**
- * Struct containing information about a client of the service
- */
-struct CadetClient;
-
-#include "gnunet-service-cadet_channel.h"
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Initialize server subsystem.
- *
- * @param handle Server handle.
- */
-void
-GML_init (struct GNUNET_SERVER_Handle *handle);
-
-/**
- * Install server (service) handlers and start listening to clients.
- */
-void
-GML_start (void);
-
-/**
- * Shutdown server.
- */
-void
-GML_shutdown (void);
-
-/**
- * Get a channel from a client.
- *
- * @param c Client to check.
- * @param ccn Channel ID, must be local (> 0x800...).
- *
- * @return non-NULL if channel exists in the clients lists
- */
-struct CadetChannel *
-GML_channel_get (struct CadetClient *c,
-                 struct GNUNET_CADET_ClientChannelNumber ccn);
-
-/**
- * Add a channel to a client
- *
- * @param client Client.
- * @param ccn Channel ID.
- * @param ch Channel.
- */
-void
-GML_channel_add (struct CadetClient *client,
-                 struct GNUNET_CADET_ClientChannelNumber ccn,
-                 struct CadetChannel *ch);
-
-/**
- * Remove a channel from a client
- *
- * @param client Client.
- * @param ccn Channel ID.
- * @param ch Channel.
- */
-void
-GML_channel_remove (struct CadetClient *client,
-                    struct GNUNET_CADET_ClientChannelNumber ccn,
-                    struct CadetChannel *ch);
-
-/**
- * Get the tunnel's next free local channel ID.
- *
- * @param c Client.
- *
- * @return LID of a channel free to use.
- */
-struct GNUNET_CADET_ClientChannelNumber
-GML_get_next_ccn (struct CadetClient *c);
-
-/**
- * Check if client has registered with the service and has not disconnected
- *
- * @param client the client to check
- *
- * @return non-NULL if client exists in the global DLL
- */
-struct CadetClient *
-GML_client_get (struct GNUNET_SERVER_Client *client);
-
-/**
- * Find a client that has opened a port
- *
- * @param port Port to check.
- *
- * @return non-NULL if a client has the port.
- */
-struct CadetClient *
-GML_client_get_by_port (const struct GNUNET_HashCode *port);
-
-/**
- * Deletes a tunnel from a client (either owner or destination).
- *
- * @param c Client whose tunnel to delete.
- * @param ch Channel which should be deleted.
- * @param id Channel ID.
- */
-void
-GML_client_delete_channel (struct CadetClient *c,
-                           struct CadetChannel *ch,
-                           struct GNUNET_CADET_ClientChannelNumber id);
-
-/**
- * Build a local ACK message and send it to a local client, if needed.
- *
- * If the client was already allowed to send data, do nothing.
- *
- * @param c Client to whom send the ACK.
- * @param id Channel ID to use
- */
-void
-GML_send_ack (struct CadetClient *c,
-              struct GNUNET_CADET_ClientChannelNumber id);
-
-/**
- * Notify the appropriate client that a new incoming channel was created.
- *
- * @param c Client to notify.
- * @param id Channel ID.
- * @param port Channel's destination port.
- * @param opt Options (bit array).
- * @param peer Origin peer.
- */
-void
-GML_send_channel_create (struct CadetClient *c,
-                         struct GNUNET_CADET_ClientChannelNumber id,
-                         const struct GNUNET_HashCode *port,
-                         uint32_t opt,
-                         const struct GNUNET_PeerIdentity *peer);
-
-/**
- * Build a local channel NACK message and send it to a local client.
- *
- * @param c Client to whom send the NACK.
- * @param id Channel ID to use
- */
-void
-GML_send_channel_nack (struct CadetClient *c,
-                       struct GNUNET_CADET_ClientChannelNumber id);
-
-
-/**
- * Notify a client that a channel is no longer valid.
- *
- * @param c Client.
- * @param id ID of the channel that is destroyed.
- */
-void
-GML_send_channel_destroy (struct CadetClient *c,
-                          struct GNUNET_CADET_ClientChannelNumber id);
-
-
-/**
- * Modify the cadet message ID from global to local and send to client.
- *
- * @param c Client to send to.
- * @param msg Message to modify and send.
- * @param id Channel ID to use (c can be both owner and client).
- */
-void
-GML_send_data (struct CadetClient *c,
-               const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-               struct GNUNET_CADET_ClientChannelNumber id);
-
-/**
- * Get the static string to represent a client.
- *
- * @param c Client.
- *
- * @return Static string for the client.
- */
-const char *
-GML_2s (const struct CadetClient *c);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_CADET_SERVICE_LOCAL_H */
-#endif
-/* end of gnunet-cadet-service_LOCAL.h */
diff --git a/src/cadet/gnunet-service-cadet-new_paths.c b/src/cadet/gnunet-service-cadet_paths.c
index c6121a133..13752643c 100644
--- a/src/cadet/gnunet-service-cadet-new_paths.c
+++ b/src/cadet/gnunet-service-cadet_paths.c
@@ -18,16 +18,16 @@
      Boston, MA 02110-1301, USA.
 */
 /**
- * @file cadet/gnunet-service-cadet-new_paths.c
+ * @file cadet/gnunet-service-cadet_paths.c
  * @brief Information we track per path.
  * @author Bartlomiej Polot
  * @author Christian Grothoff
  */
 #include "platform.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
 
 
 #define LOG(level, ...) GNUNET_log_from(level,"cadet-pat",__VA_ARGS__)
diff --git a/src/cadet/gnunet-service-cadet-new_paths.h b/src/cadet/gnunet-service-cadet_paths.h
index 7310d75e6..6b7bef640 100644
--- a/src/cadet/gnunet-service-cadet-new_paths.h
+++ b/src/cadet/gnunet-service-cadet_paths.h
@@ -29,7 +29,7 @@
 #define GNUNET_SERVICE_CADET_PATHS_H
 
 #include "gnunet_util_lib.h"
-#include "gnunet-service-cadet-new.h"
+#include "gnunet-service-cadet.h"
 
 /**
  * Create a peer path based on the result of a DHT lookup.  If we
diff --git a/src/cadet/gnunet-service-cadet_peer.c b/src/cadet/gnunet-service-cadet_peer.c
index fa3f2be80..71c7c67d0 100644
--- a/src/cadet/gnunet-service-cadet_peer.c
+++ b/src/cadet/gnunet-service-cadet_peer.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013, 2015 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,2193 +17,1461 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
+
 /**
  * @file cadet/gnunet-service-cadet_peer.c
- * @brief GNUnet CADET service connection handling
+ * @brief Information we track per peer.
  * @author Bartlomiej Polot
+ * @author Christian Grothoff
+ *
+ * TODO:
+ * - optimize stopping/restarting DHT search to situations
+ *   where we actually need it (i.e. not if we have a direct connection,
+ *   or if we already have plenty of good short ones, or maybe even
+ *   to take a break if we have some connections and have searched a lot (?))
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
+#include "gnunet_hello_lib.h"
 #include "gnunet_signatures.h"
 #include "gnunet_transport_service.h"
 #include "gnunet_ats_service.h"
 #include "gnunet_core_service.h"
 #include "gnunet_statistics_service.h"
 #include "cadet_protocol.h"
-#include "gnunet-service-cadet_peer.h"
-#include "gnunet-service-cadet_dht.h"
 #include "gnunet-service-cadet_connection.h"
-#include "gnunet-service-cadet_tunnel.h"
-#include "cadet_path.h"
+#include "gnunet-service-cadet_dht.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
+#include "gnunet-service-cadet_tunnels.h"
 
-#define LOG(level, ...) GNUNET_log_from (level,"cadet-p2p",__VA_ARGS__)
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-p2p",__VA_ARGS__)
 
+#define LOG(level, ...) GNUNET_log_from(level,"cadet-per",__VA_ARGS__)
 
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
 
 /**
- * Information about a queued message on the peer level.
+ * How long do we wait until tearing down an idle peer?
  */
-struct CadetPeerQueue {
-
-    struct CadetPeerQueue *next;
-    struct CadetPeerQueue *prev;
-
-    /**
-     * Envelope to cancel message before MQ sends it.
-     */
-    struct GNUNET_MQ_Envelope *env;
-
-    /**
-     * Peer (neighbor) this message is being sent to.
-     */
-    struct CadetPeer *peer;
-
-    /**
-     * Continuation to call to notify higher layers about message sent.
-     */
-    GCP_sent cont;
-
-    /**
-     * Closure for @a cont.
-     */
-    void *cont_cls;
-
-    /**
-     * Task to asynchronously run the drop continuation.
-     */
-    struct GNUNET_SCHEDULER_Task *drop_task;
-
-    /**
-     * Time when message was queued for sending.
-     */
-    struct GNUNET_TIME_Absolute queue_timestamp;
-
-    /**
-     * #GNUNET_YES if message was management traffic (POLL, ACK, ...).
-     */
-    int management_traffic;
-
-    /**
-     * Message type.
-     */
-    uint16_t type;
-
-    /**
-     * Message size.
-     */
-    uint16_t size;
-
-    /**
-     * Type of the message's payload, if it was encrypted data.
-     */
-    uint16_t payload_type;
-
-    /**
-     * ID of the payload (PID, ACK #, ...).
-     */
-    struct CadetEncryptedMessageIdentifier payload_id;
-
-    /**
-     * Connection this message was sent on.
-     */
-    struct CadetConnection *c;
-
-    /**
-     * Direction in @a c this message was send on (#GNUNET_YES = FWD).
-     */
-    int c_fwd;
-};
-
+#define IDLE_PEER_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
 
 /**
- * Struct containing all information regarding a given peer
+ * How long do we keep paths around if we no longer care about the peer?
  */
-struct CadetPeer
-{
-    /**
-     * ID of the peer
-     */
-    GNUNET_PEER_Id id;
-
-    struct CadetPeerQueue *q_head;
-    struct CadetPeerQueue *q_tail;
-
-    /**
-     * Last time we heard from this peer
-     */
-    struct GNUNET_TIME_Absolute last_contact;
-
-    /**
-     * Paths to reach the peer, ordered by ascending hop count
-     */
-    struct CadetPeerPath *path_head;
-
-    /**
-     * Paths to reach the peer, ordered by ascending hop count
-     */
-    struct CadetPeerPath *path_tail;
-
-    /**
-     * Handle to stop the DHT search for paths to this peer
-     */
-    struct GCD_search_handle *search_h;
-
-    /**
-     * Handle to stop the DHT search for paths to this peer
-     */
-    struct GNUNET_SCHEDULER_Task *search_delayed;
-
-    /**
-     * Tunnel to this peer, if any.
-     */
-    struct CadetTunnel *tunnel;
-
-    /**
-     * Connections that go through this peer; indexed by tid.
-     */
-    struct GNUNET_CONTAINER_MultiShortmap  *connections;
-
-    /**
-     * Handle for core transmissions.
-     */
-    struct GNUNET_MQ_Handle *core_mq;
-
-    /**
-     * How many messages are in the queue to this peer.
-     */
-    unsigned int queue_n;
-
-    /**
-     * Hello message.
-     */
-    struct GNUNET_HELLO_Message* hello;
-
-    /**
-     * Handle to us offering the HELLO to the transport.
-     */
-    struct GNUNET_TRANSPORT_OfferHelloHandle *hello_offer;
-
-    /**
-     * Handle to our ATS request asking ATS to suggest an address
-     * to TRANSPORT for this peer (to establish a direct link).
-     */
-    struct GNUNET_ATS_ConnectivitySuggestHandle *connectivity_suggestion;
+#define IDLE_PATH_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 2)
 
-};
-
-
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
 
-/**
- * Local peer own ID (full value).
- */
-extern struct GNUNET_PeerIdentity my_full_id;
 
-/**
- * Local peer own ID (short)
- */
-extern GNUNET_PEER_Id myid;
 
 /**
- * Peers known, indexed by PeerIdentity, values of type `struct CadetPeer`.
+ * Data structure used to track whom we have to notify about changes
+ * to our message queue.
  */
-static struct GNUNET_CONTAINER_MultiPeerMap *peers;
-
-/**
- * How many peers do we want to remember?
- */
-static unsigned long long max_peers;
+struct GCP_MessageQueueManager
+{
 
-/**
- * Percentage of messages that will be dropped (for test purposes only).
- */
-static unsigned long long drop_percent;
+  /**
+   * Kept in a DLL.
+   */
+  struct GCP_MessageQueueManager *next;
 
-/**
- * Handle to communicate with CORE.
- */
-static struct GNUNET_CORE_Handle *core_handle;
+  /**
+   * Kept in a DLL.
+   */
+  struct GCP_MessageQueueManager *prev;
 
-/**
- * Our configuration;
- */
-static const struct GNUNET_CONFIGURATION_Handle *cfg;
+  /**
+   * Function to call with updated message queue object.
+   */
+  GCP_MessageQueueNotificationCallback cb;
 
-/**
- * Handle to communicate with ATS.
- */
-static struct GNUNET_ATS_ConnectivityHandle *ats_ch;
+  /**
+   * Closure for @e cb.
+   */
+  void *cb_cls;
 
-/**
- * Shutdown falg.
- */
-static int in_shutdown;
+  /**
+   * The peer this is for.
+   */
+  struct CadetPeer *cp;
 
+  /**
+   * Envelope this manager would like to transmit once it is its turn.
+   */
+  struct GNUNET_MQ_Envelope *env;
 
-/******************************************************************************/
-/*****************************  CORE HELPERS  *********************************/
-/******************************************************************************/
+};
 
 
 /**
- * Iterator to notify all connections of a broken link. Mark connections
- * to destroy after all traffic has been sent.
- *
- * @param cls Closure (disconnected peer).
- * @param key Current key code (peer id).
- * @param value Value in the hash map (connection).
- *
- * @return #GNUNET_YES to continue to iterate.
+ * Struct containing all information regarding a given peer
  */
-static int
-notify_broken (void *cls,
-               const struct GNUNET_ShortHashCode *key,
-               void *value)
+struct CadetPeer
 {
-    struct CadetPeer *peer = cls;
-    struct CadetConnection *c = value;
+  /**
+   * ID of the peer
+   */
+  struct GNUNET_PeerIdentity pid;
+
+  /**
+   * Last time we heard from this peer (currently not used!)
+   */
+  struct GNUNET_TIME_Absolute last_contactXXX;
+
+  /**
+   * Array of DLLs of paths traversing the peer, organized by the
+   * offset of the peer on the larger path.
+   */
+  struct CadetPeerPathEntry **path_heads;
+
+  /**
+   * Array of DLL of paths traversing the peer, organized by the
+   * offset of the peer on the larger path.
+   */
+  struct CadetPeerPathEntry **path_tails;
+
+  /**
+   * Notifications to call when @e core_mq changes.
+   */
+  struct GCP_MessageQueueManager *mqm_head;
+
+  /**
+   * Notifications to call when @e core_mq changes.
+   */
+  struct GCP_MessageQueueManager *mqm_tail;
+
+  /**
+   * Pointer to first "ready" entry in @e mqm_head.
+   */
+  struct GCP_MessageQueueManager *mqm_ready_ptr;
+
+  /**
+   * MIN-heap of paths owned by this peer (they also end at this
+   * peer).  Ordered by desirability.
+   */
+  struct GNUNET_CONTAINER_Heap *path_heap;
+
+  /**
+   * Handle to stop the DHT search for paths to this peer
+   */
+  struct GCD_search_handle *search_h;
+
+  /**
+   * Task to clean up @e path_heap asynchronously.
+   */
+  struct GNUNET_SCHEDULER_Task *heap_cleanup_task;
+
+  /**
+   * Task to destroy this entry.
+   */
+  struct GNUNET_SCHEDULER_Task *destroy_task;
+
+  /**
+   * Tunnel to this peer, if any.
+   */
+  struct CadetTunnel *t;
+
+  /**
+   * Connections that go through this peer; indexed by tid.
+   */
+  struct GNUNET_CONTAINER_MultiShortmap *connections;
+
+  /**
+   * Handle for core transmissions.
+   */
+  struct GNUNET_MQ_Handle *core_mq;
+
+  /**
+   * Hello message of the peer.
+   */
+  struct GNUNET_HELLO_Message *hello;
+
+  /**
+   * Handle to us offering the HELLO to the transport.
+   */
+  struct GNUNET_TRANSPORT_OfferHelloHandle *hello_offer;
+
+  /**
+   * Handle to our ATS request asking ATS to suggest an address
+   * to TRANSPORT for this peer (to establish a direct link).
+   */
+  struct GNUNET_ATS_ConnectivitySuggestHandle *connectivity_suggestion;
+
+  /**
+   * How many messages are in the queue to this peer.
+   */
+  unsigned int queue_n;
+
+  /**
+   * How many paths do we have to this peer (in all @e path_heads DLLs combined).
+   */
+  unsigned int num_paths;
+
+  /**
+   * Sum over all of the offsets of all of the paths in the @a path_heads DLLs.
+   * Used to speed-up @GCP_get_desirability_of_path() calculation.
+   */
+  unsigned int off_sum;
+
+  /**
+   * Number of message queue managers of this peer that have a message in waiting.
+   *
+   * Used to quickly see if we need to bother scanning the @e msm_head DLL.
+   * TODO: could be replaced by another DLL that would then allow us to avoid
+   * the O(n)-scan of the DLL for ready entries!
+   */
+  unsigned int mqm_ready_counter;
+
+  /**
+   * Current length of the @e path_heads and @path_tails arrays.
+   * The arrays should be grown as needed.
+   */
+  unsigned int path_dll_length;
 
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Notifying %s due to %s disconnect\n",
-         GCC_2s (c), GCP_2s (peer));
-    GCC_neighbor_disconnected (c, peer);
-    return GNUNET_YES;
-}
+};
 
 
 /**
- * Remove the direct path to the peer.
- *
- * @param peer Peer to remove the direct path from.
- */
-static struct CadetPeerPath *
-pop_direct_path (struct CadetPeer *peer)
-{
-    struct CadetPeerPath *iter;
-
-    for (iter = peer->path_head; NULL != iter; iter = iter->next)
-    {
-        if (2 >= iter->length)
-        {
-            GNUNET_CONTAINER_DLL_remove (peer->path_head,
-                                         peer->path_tail,
-                                         iter);
-            return iter;
-        }
-    }
-    return NULL;
-}
-
-/**
- * Call the continuation after a message has been sent or dropped.
- *
- * This funcion removes the message from the queue.
+ * Get the static string for a peer ID.
  *
- * @param q Queue handle.
- * @param sent #GNUNET_YES if was sent to CORE, #GNUNET_NO if dropped.
+ * @param cp Peer.
+ * @return Static string for it's ID.
  */
-static void
-call_peer_cont (struct CadetPeerQueue *q, int sent);
+const char *
+GCP_2s (const struct CadetPeer *cp)
+{
+  static char buf[32];
+
+  GNUNET_snprintf (buf,
+                   sizeof (buf),
+                   "P(%s)",
+                   GNUNET_i2s (&cp->pid));
+  return buf;
+}
+
+
+/**
+ * Calculate how desirable a path is for @a cp if @a cp
+ * is at offset @a off.
+ *
+ * The 'desirability_table.c' program can be used to compute a list of
+ * sample outputs for different scenarios.  Basically, we score paths
+ * lower if there are many alternatives, and higher if they are
+ * shorter than average, and very high if they are much shorter than
+ * average and without many alternatives.
+ *
+ * @param cp a peer reachable via a path
+ * @param off offset of @a cp in the path
+ * @return score how useful a path is to reach @a cp,
+ *         positive scores mean path is more desirable
+ */
+double
+GCP_get_desirability_of_path (struct CadetPeer *cp,
+                              unsigned int off)
+{
+  unsigned int num_alts = cp->num_paths;
+  unsigned int off_sum;
+  double avg_sum;
+  double path_delta;
+  double weight_alts;
+
+  GNUNET_assert (num_alts >= 1); /* 'path' should be in there! */
+  GNUNET_assert (0 != cp->path_dll_length);
+
+  /* We maintain 'off_sum' in 'peer' and thereby
+     avoid the SLOW recalculation each time. Kept here
+     just to document what is going on. */
+#if SLOW
+  off_sum = 0;
+  for (unsigned int j=0;j<cp->path_dll_length;j++)
+    for (struct CadetPeerPathEntry *pe = cp->path_heads[j];
+         NULL != pe;
+         pe = pe->next)
+      off_sum += j;
+  GNUNET_assert (off_sum == cp->off_sum);
+#else
+  off_sum = cp->off_sum;
+#endif
+  avg_sum = off_sum * 1.0 / cp->path_dll_length;
+  path_delta = off - avg_sum;
+  /* path_delta positiv: path off of peer above average (bad path for peer),
+     path_delta negativ: path off of peer below average (good path for peer) */
+  if (path_delta <= - 1.0)
+    weight_alts = - num_alts / path_delta; /* discount alternative paths */
+  else if (path_delta >= 1.0)
+    weight_alts = num_alts * path_delta; /* overcount alternative paths */
+  else
+    weight_alts = num_alts; /* count alternative paths normally */
 
 
-/******************************************************************************/
-/***************************** CORE CALLBACKS *********************************/
-/******************************************************************************/
+  /* off+1: long paths are generally harder to find and thus count
+     a bit more as they get longer.  However, above-average paths
+     still need to count less, hence the squaring of that factor. */
+  return (off + 1.0) / (weight_alts * weight_alts);
+}
 
 
 /**
- * Method called whenever a given peer connects.
+ * This peer is no longer be needed, clean it up now.
  *
- * @param cls Core closure (unused).
- * @param peer Peer identity this notification is about
- * @param mq Message Queue to this peer.
- *
- * @return Internal closure for handlers (CadetPeer struct).
+ * @param cls peer to clean up
  */
-static void *
-core_connect_handler (void *cls,
-                      const struct GNUNET_PeerIdentity *peer,
-                      struct GNUNET_MQ_Handle *mq)
-{
-    struct CadetPeer *neighbor;
-    struct CadetPeerPath *path;
-    char own_id[16];
-
-    GCC_check_connections ();
-    GNUNET_snprintf (own_id,
-                     sizeof (own_id),
-                     "%s",
-                     GNUNET_i2s (&my_full_id));
-
-    /* Save a path to the neighbor */
-    neighbor = GCP_get (peer, GNUNET_YES);
-    if (myid == neighbor->id)
-    {
-        LOG (GNUNET_ERROR_TYPE_INFO,
-             "CONNECTED %s (self)\n",
-             own_id);
-        path = path_new (1);
-    }
-    else
-    {
-        LOG (GNUNET_ERROR_TYPE_INFO,
-             "CONNECTED %s <= %s\n",
-             own_id,
-             GNUNET_i2s (peer));
-        path = path_new (2);
-        path->peers[1] = neighbor->id;
-        GNUNET_PEER_change_rc (neighbor->id, 1);
-        GNUNET_assert (NULL == neighbor->core_mq);
-        neighbor->core_mq = mq;
-    }
-    path->peers[0] = myid;
-    GNUNET_PEER_change_rc (myid, 1);
-    GCP_add_path (neighbor, path, GNUNET_YES);
-
-    /* Create the connections hashmap */
-    GNUNET_assert (NULL == neighbor->connections);
-    neighbor->connections = GNUNET_CONTAINER_multishortmap_create (16,
-                                                                   GNUNET_YES);
-    GNUNET_STATISTICS_update (stats,
-                              "# peers",
-                              1,
-                              GNUNET_NO);
-
-    if ( (NULL != GCP_get_tunnel (neighbor)) &&
-            (0 > GNUNET_CRYPTO_cmp_peer_identity (&my_full_id, peer)) )
-    {
-        GCP_connect (neighbor);
-    }
-    GCC_check_connections ();
+static void
+destroy_peer (void *cls)
+{
+  struct CadetPeer *cp = cls;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying state about peer %s\n",
+       GCP_2s (cp));
+  cp->destroy_task = NULL;
+  GNUNET_assert (NULL == cp->t);
+  GNUNET_assert (NULL == cp->core_mq);
+  GNUNET_assert (0 == cp->num_paths);
+  for (unsigned int i=0;i<cp->path_dll_length;i++)
+    GNUNET_assert (NULL == cp->path_heads[i]);
+  GNUNET_assert (0 == GNUNET_CONTAINER_multishortmap_size (cp->connections));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multipeermap_remove (peers,
+                                                       &cp->pid,
+                                                       cp));
+  GNUNET_free_non_null (cp->path_heads);
+  GNUNET_free_non_null (cp->path_tails);
+  cp->path_dll_length = 0;
+  if (NULL != cp->search_h)
+  {
+    GCD_search_stop (cp->search_h);
+    cp->search_h = NULL;
+  }
+  /* FIXME: clean up search_delayedXXX! */
 
-    return neighbor;
+  if (NULL != cp->hello_offer)
+  {
+    GNUNET_TRANSPORT_offer_hello_cancel (cp->hello_offer);
+    cp->hello_offer = NULL;
+  }
+  if (NULL != cp->connectivity_suggestion)
+  {
+    GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
+    cp->connectivity_suggestion = NULL;
+  }
+  GNUNET_CONTAINER_multishortmap_destroy (cp->connections);
+  if (NULL != cp->path_heap)
+  {
+    GNUNET_CONTAINER_heap_destroy (cp->path_heap);
+    cp->path_heap = NULL;
+  }
+  if (NULL != cp->heap_cleanup_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->heap_cleanup_task);
+    cp->heap_cleanup_task = NULL;
+  }
+  GNUNET_free_non_null (cp->hello);
+  /* Peer should not be freed if paths exist; if there are no paths,
+     there ought to be no connections, and without connections, no
+     notifications. Thus we can assert that mqm_head is empty at this
+     point. */
+  GNUNET_assert (NULL == cp->mqm_head);
+  GNUNET_assert (NULL == cp->mqm_ready_ptr);
+  GNUNET_free (cp);
 }
 
 
 /**
- * Method called whenever a peer disconnects.
+ * This peer is now on more "active" duty, activate processes related to it.
  *
- * @param cls Core closure (unused).
- * @param peer Peer identity this notification is about.
- * @param internal_cls Internal closure (CadetPeer struct).
+ * @param cp the more-active peer
  */
 static void
-core_disconnect_handler (void *cls,
-                         const struct GNUNET_PeerIdentity *peer,
-                         void *internal_cls)
+consider_peer_activate (struct CadetPeer *cp)
 {
-    struct CadetPeer *p = internal_cls;
-    struct CadetPeerPath *direct_path;
-    char own_id[16];
-
-    GCC_check_connections ();
-    strncpy (own_id, GNUNET_i2s (&my_full_id), 16);
-    own_id[15] = '\0';
-    if (myid == p->id)
-    {
-        LOG (GNUNET_ERROR_TYPE_INFO,
-             "DISCONNECTED %s (self)\n",
-             own_id);
-    }
-    else
+  uint32_t strength;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Updating peer %s activation state (%u connections)%s%s\n",
+       GCP_2s (cp),
+       GNUNET_CONTAINER_multishortmap_size (cp->connections),
+       (NULL == cp->t) ? "" : " with tunnel",
+       (NULL == cp->core_mq) ? "" : " with CORE link");
+  if (NULL != cp->destroy_task)
+  {
+    /* It's active, do not destory! */
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
+  if ( (0 == GNUNET_CONTAINER_multishortmap_size (cp->connections)) &&
+       (NULL == cp->t) )
+  {
+    /* We're just on a path or directly connected; don't bother too much */
+    if (NULL != cp->connectivity_suggestion)
     {
-        LOG (GNUNET_ERROR_TYPE_INFO,
-             "DISCONNECTED %s <= %s\n",
-             own_id, GNUNET_i2s (peer));
-        p->core_mq = NULL;
+      GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
+      cp->connectivity_suggestion = NULL;
     }
-    direct_path = pop_direct_path (p);
-    if (NULL != p->connections)
+    if (NULL != cp->search_h)
     {
-      GNUNET_CONTAINER_multishortmap_iterate (p->connections,
-                                              &notify_broken,
-                                              p);
-      GNUNET_CONTAINER_multishortmap_destroy (p->connections);
-      p->connections = NULL;
+      GCD_search_stop (cp->search_h);
+      cp->search_h = NULL;
     }
-    GNUNET_STATISTICS_update (stats,
-                              "# peers",
-                              -1,
-                              GNUNET_NO);
-    path_destroy (direct_path);
-    GCC_check_connections ();
-}
-
-
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-
-/**
- * Check if the create_connection message has the appropriate size.
- *
- * @param cls Closure (unused).
- * @param msg Message to check.
- *
- * @return #GNUNET_YES if size is correct, #GNUNET_NO otherwise.
- */
-static int
-check_create (void *cls, const struct GNUNET_CADET_ConnectionCreateMessage *msg)
-{
-    uint16_t size;
-
-    size = ntohs (msg->header.size);
-    if (size < sizeof (*msg))
+    return;
+  }
+  if (NULL == cp->core_mq)
+  {
+    /* Lacks direct connection, try to create one by querying the DHT */
+    if ( (NULL == cp->search_h) &&
+         (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) )
+      cp->search_h
+        = GCD_search (&cp->pid);
+  }
+  else
+  {
+    /* Have direct connection, stop DHT search if active */
+    if (NULL != cp->search_h)
     {
-        GNUNET_break_op (0);
-        return GNUNET_NO;
+      GCD_search_stop (cp->search_h);
+      cp->search_h = NULL;
     }
-    return GNUNET_YES;
-}
-
-/**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE
- *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
- */
-static void
-handle_create (void *cls, const struct GNUNET_CADET_ConnectionCreateMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_create (peer, msg);
-}
-
-
-/**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK
- *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
- */
-static void
-handle_confirm (void *cls, const struct GNUNET_CADET_ConnectionCreateAckMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_confirm (peer, msg);
-}
-
+  }
 
-/**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN
- *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
- */
-static void
-handle_broken (void *cls, const struct GNUNET_CADET_ConnectionBrokenMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_broken (peer, msg);
+  /* If we have a tunnel, our urge for connections is much bigger */
+  strength = (NULL != cp->t) ? 32 : 1;
+  if (NULL != cp->connectivity_suggestion)
+    GNUNET_ATS_connectivity_suggest_cancel (cp->connectivity_suggestion);
+  cp->connectivity_suggestion
+    = GNUNET_ATS_connectivity_suggest (ats_ch,
+                                       &cp->pid,
+                                       strength);
 }
 
 
 /**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY
+ * This peer may no longer be needed, consider cleaning it up.
  *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
+ * @param cp peer to clean up
  */
 static void
-handle_destroy (void *cls, const struct GNUNET_CADET_ConnectionDestroyMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_destroy (peer, msg);
-}
+consider_peer_destroy (struct CadetPeer *cp);
 
 
 /**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK
+ * We really no longere care about a peer, stop hogging memory with paths to it.
+ * Afterwards, see if there is more to be cleaned up about this peer.
  *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
+ * @param cls a `struct CadetPeer`.
  */
 static void
-handle_ack (void *cls, const struct GNUNET_CADET_ConnectionEncryptedAckMessage *msg)
+drop_paths (void *cls)
 {
-    struct CadetPeer *peer = cls;
-    GCC_handle_ack (peer, msg);
-}
+  struct CadetPeer *cp = cls;
+  struct CadetPeerPath *path;
 
-
-/**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL
- *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
- */
-static void
-handle_poll (void *cls, const struct GNUNET_CADET_ConnectionHopByHopPollMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_poll (peer, msg);
+  cp->destroy_task = NULL;
+  while (NULL != (path = GNUNET_CONTAINER_heap_remove_root (cp->path_heap)))
+    GCPP_release (path);
+  consider_peer_destroy (cp);
 }
 
 
 /**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX
+ * This peer may no longer be needed, consider cleaning it up.
  *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
+ * @param cp peer to clean up
  */
 static void
-handle_kx (void *cls, const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_kx (peer, msg);
-}
-
-
-/**
- * Check if the encrypted message has the appropriate size.
- *
- * @param cls Closure (unused).
- * @param msg Message to check.
- *
- * @return #GNUNET_YES if size is correct, #GNUNET_NO otherwise.
- */
-static int
-check_encrypted (void *cls, const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
+consider_peer_destroy (struct CadetPeer *cp)
 {
-    uint16_t size;
-    uint16_t minimum_size;
-
-    size = ntohs (msg->header.size);
-    minimum_size = sizeof (struct GNUNET_CADET_TunnelEncryptedMessage)
-                   + sizeof (struct GNUNET_MessageHeader);
-
-    if (size < minimum_size)
-    {
-        GNUNET_break_op (0);
-        return GNUNET_NO;
-    }
-    return GNUNET_YES;
-}
+  struct GNUNET_TIME_Relative exp;
 
-/**
- * Handle for #GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED.
- *
- * @param cls Closure (CadetPeer for neighbor that sent the message).
- * @param msg Message itself.
- */
-static void
-handle_encrypted (void *cls, const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
-{
-    struct CadetPeer *peer = cls;
-    GCC_handle_encrypted (peer, msg);
-}
-
-
-/**
- * To be called on core init/fail.
- *
- * @param cls Closure (config)
- * @param identity The public identity of this peer.
- */
-static void
-core_init_notify (void *cls,
-                  const struct GNUNET_PeerIdentity *identity);
-
-
-static void
-connect_to_core (const struct GNUNET_CONFIGURATION_Handle *c)
-{
-    struct GNUNET_MQ_MessageHandler core_handlers[] = {
-        GNUNET_MQ_hd_var_size (create,
-                               GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE,
-                               struct GNUNET_CADET_ConnectionCreateMessage,
-                               NULL),
-        GNUNET_MQ_hd_fixed_size (confirm,
-                                 GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK,
-                                 struct GNUNET_CADET_ConnectionCreateAckMessage,
-                                 NULL),
-        GNUNET_MQ_hd_fixed_size (broken,
-                                GNUNET_MESSAGE_TYPE_CADET_CONNECTION_BROKEN,
-                                struct GNUNET_CADET_ConnectionBrokenMessage,
-                                NULL),
-        GNUNET_MQ_hd_fixed_size (destroy,
-                                 GNUNET_MESSAGE_TYPE_CADET_CONNECTION_DESTROY,
-                                 struct GNUNET_CADET_ConnectionDestroyMessage,
-                                 NULL),
-        GNUNET_MQ_hd_fixed_size (ack,
-                                 GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK,
-                                 struct GNUNET_CADET_ConnectionEncryptedAckMessage,
-                                 NULL),
-        GNUNET_MQ_hd_fixed_size (poll,
-                                 GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL,
-                                 struct GNUNET_CADET_ConnectionHopByHopPollMessage,
-                                 NULL),
-        GNUNET_MQ_hd_fixed_size (kx,
-                                 GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX,
-                                 struct GNUNET_CADET_TunnelKeyExchangeMessage,
-                                 NULL),
-        GNUNET_MQ_hd_var_size (encrypted,
-                               GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED,
-                               struct GNUNET_CADET_TunnelEncryptedMessage,
-                               NULL),
-        GNUNET_MQ_handler_end ()
-    };
-    core_handle = GNUNET_CORE_connect (c, NULL,
-                                       &core_init_notify,
-                                       &core_connect_handler,
-                                       &core_disconnect_handler,
-                                       core_handlers);
-}
-
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
-
-/**
- * To be called on core init/fail.
- *
- * @param cls Closure (config)
- * @param identity The public identity of this peer.
- */
-static void
-core_init_notify (void *cls,
-                  const struct GNUNET_PeerIdentity *core_identity)
-{
-    const struct GNUNET_CONFIGURATION_Handle *c = cls;
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Core init\n");
-    if (0 != memcmp (core_identity, &my_full_id, sizeof (my_full_id)))
-    {
-        LOG (GNUNET_ERROR_TYPE_ERROR, _("Wrong CORE service\n"));
-        LOG (GNUNET_ERROR_TYPE_ERROR, " core id %s\n", GNUNET_i2s (core_identity));
-        LOG (GNUNET_ERROR_TYPE_ERROR, " my id %s\n", GNUNET_i2s (&my_full_id));
-        GNUNET_CORE_disconnect (core_handle);
-        connect_to_core (c);
-        return;
-    }
-    GML_start ();
+  if (NULL != cp->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
+  if (NULL != cp->t)
+    return; /* still relevant! */
+  if (NULL != cp->core_mq)
+    return; /* still relevant! */
+  if (0 != GNUNET_CONTAINER_multishortmap_size (cp->connections))
+    return; /* still relevant! */
+  if ( (NULL != cp->path_heap) &&
+       (0 < GNUNET_CONTAINER_heap_get_size (cp->path_heap)) )
+  {
+    cp->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_PATH_TIMEOUT,
+                                                     &drop_paths,
+                                                     cp);
+    return;
+  }
+  if (0 != cp->num_paths)
+    return; /* still relevant! */
+  if (NULL != cp->hello)
+  {
+    /* relevant only until HELLO expires */
+    exp = GNUNET_TIME_absolute_get_remaining (GNUNET_HELLO_get_last_expiration (cp->hello));
+    cp->destroy_task = GNUNET_SCHEDULER_add_delayed (exp,
+                                                     &destroy_peer,
+                                                     cp);
+    return;
+  }
+  cp->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_PEER_TIMEOUT,
+                                                   &destroy_peer,
+                                                   cp);
 }
 
 
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
-
 /**
- * Get priority for a queued message.
- *
- * @param q Queued message
- *
- * @return CORE priority to use.
+ * Set the message queue to @a mq for peer @a cp and notify watchers.
  *
- * FIXME make static
- * FIXME use when sending
+ * @param cp peer to modify
+ * @param mq message queue to set (can be NULL)
  */
-enum GNUNET_CORE_Priority
-get_priority (struct CadetPeerQueue *q)
-{
-    enum GNUNET_CORE_Priority low;
-    enum GNUNET_CORE_Priority high;
-
-    if (NULL == q)
-    {
-        GNUNET_break (0);
-        return GNUNET_CORE_PRIO_BACKGROUND;
-    }
-
-    /* Relayed traffic has lower priority, our own traffic has higher */
-    if (NULL == q->c || GNUNET_NO == GCC_is_origin (q->c, q->c_fwd))
+void
+GCP_set_mq (struct CadetPeer *cp,
+            struct GNUNET_MQ_Handle *mq)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Message queue for peer %s is now %p\n",
+       GCP_2s (cp),
+       mq);
+  cp->core_mq = mq;
+  for (struct GCP_MessageQueueManager *mqm = cp->mqm_head, *next;
+       NULL != mqm;
+       mqm = next)
+  {
+    /* Save next pointer in case mqm gets freed by the callback */
+    next = mqm->next;
+    if (NULL == mq)
     {
-        low = GNUNET_CORE_PRIO_BEST_EFFORT;
-        high = GNUNET_CORE_PRIO_URGENT;
+      if (NULL != mqm->env)
+      {
+        GNUNET_MQ_discard (mqm->env);
+        mqm->env = NULL;
+        mqm->cb (mqm->cb_cls,
+                 GNUNET_SYSERR);
+      }
+      else
+      {
+        mqm->cb (mqm->cb_cls,
+                 GNUNET_NO);
+      }
     }
     else
     {
-        low = GNUNET_CORE_PRIO_URGENT;
-        high = GNUNET_CORE_PRIO_CRITICAL_CONTROL;
+      GNUNET_assert (NULL == mqm->env);
+      mqm->cb (mqm->cb_cls,
+               GNUNET_YES);
     }
+  }
+  if ( (NULL != mq) ||
+       (NULL != cp->t) )
+    consider_peer_activate (cp);
+  else
+    consider_peer_destroy (cp);
 
-    /* Bulky payload has lower priority, control traffic has higher. */
-    if (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED == q->type)
-        return low;
-    return high;
-}
-
-
-/**
- * Cancel all messages queued to CORE MQ towards this peer.
- *
- * @param peer Peer towards which to cancel all messages.
- */
-static void
-cancel_queued_messages (struct CadetPeer *peer)
-{
-    while (NULL != peer->q_head)
-    {
-        struct CadetPeerQueue *q;
-
-        q = peer->q_head;
-        call_peer_cont (q, GNUNET_NO);
-        GNUNET_free (q);
-    }
-}
-
-
-/**
- * Destroy the peer_info and free any allocated resources linked to it
- *
- * @param peer The peer_info to destroy.
- * @return #GNUNET_OK on success
- */
-static int
-peer_destroy (struct CadetPeer *peer)
-{
-    struct GNUNET_PeerIdentity id;
-    struct CadetPeerPath *p;
-    struct CadetPeerPath *nextp;
-
-    GNUNET_PEER_resolve (peer->id, &id);
-    GNUNET_PEER_change_rc (peer->id, -1);
-
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         "destroying peer %s\n",
-         GNUNET_i2s (&id));
-
-    if (GNUNET_YES !=
-            GNUNET_CONTAINER_multipeermap_remove (peers, &id, peer))
-    {
-        GNUNET_break (0);
-        LOG (GNUNET_ERROR_TYPE_WARNING, " peer not in peermap!!\n");
-    }
-    GCP_stop_search (peer);
-    p = peer->path_head;
-    while (NULL != p)
-    {
-        nextp = p->next;
-        GNUNET_CONTAINER_DLL_remove (peer->path_head,
-                                     peer->path_tail,
-                                     p);
-        path_destroy (p);
-        p = nextp;
-    }
-    if (NULL != peer->tunnel)
-        GCT_destroy_empty (peer->tunnel);
-    if (NULL != peer->connections)
-    {
-        GNUNET_assert (0 == GNUNET_CONTAINER_multishortmap_size (peer->connections));
-        GNUNET_CONTAINER_multishortmap_destroy (peer->connections);
-        peer->connections = NULL;
-    }
-    if (NULL != peer->hello_offer)
-    {
-        GNUNET_TRANSPORT_offer_hello_cancel (peer->hello_offer);
-        peer->hello_offer = NULL;
-    }
-    if (NULL != peer->connectivity_suggestion)
-    {
-        GNUNET_ATS_connectivity_suggest_cancel (peer->connectivity_suggestion);
-        peer->connectivity_suggestion = NULL;
-    }
-    cancel_queued_messages (peer);
+  if ( (NULL != mq) &&
+       (NULL != cp->t) )
+  {
+    /* have a new, direct path to the target, notify tunnel */
+    struct CadetPeerPath *path;
 
-    GNUNET_free_non_null (peer->hello);
-    GNUNET_free (peer);
-    return GNUNET_OK;
+    path = GCPP_get_path_from_route (1,
+                                     &cp->pid);
+    GCT_consider_path (cp->t,
+                       path,
+                       0);
+  }
 }
 
 
 /**
- * Iterator over peer hash map entries to destroy the peer during in_shutdown.
+ * Debug function should NEVER return true in production code, useful to
+ * simulate losses for testcases.
  *
- * @param cls closure
- * @param key current key code
- * @param value value in the hash map
- * @return #GNUNET_YES if we should continue to iterate,
- *         #GNUNET_NO if not.
+ * @return #GNUNET_YES or #GNUNET_NO with the decision to drop.
  */
 static int
-shutdown_peer (void *cls,
-               const struct GNUNET_PeerIdentity *key,
-               void *value)
+should_I_drop (void)
 {
-    struct CadetPeer *p = value;
-    struct CadetTunnel *t = p->tunnel;
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  shutting down %s\n", GCP_2s (p));
-    if (NULL != t)
-        GCT_destroy (t);
-    p->tunnel = NULL;
-    peer_destroy (p);
+  if (0 == drop_percent)
+    return GNUNET_NO;
+  if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                101) < drop_percent)
     return GNUNET_YES;
+  return GNUNET_NO;
 }
 
 
 /**
- * Check if peer is searching for a path (either active or delayed search).
- *
- * @param peer Peer to check
- * @return #GNUNET_YES if there is a search active.
- *         #GNUNET_NO otherwise.
- */
-static int
-is_searching (const struct CadetPeer *peer)
-{
-    return ( (NULL == peer->search_h) &&
-             (NULL == peer->search_delayed) ) ?
-           GNUNET_NO : GNUNET_YES;
-}
-
-
-/**
- * @brief Start a search for a peer.
+ * Function called when CORE took one of the messages from
+ * a message queue manager and transmitted it.
  *
- * @param cls Closure (Peer to search for).
+ * @param cls the `struct CadetPeeer` where we made progress
  */
 static void
-delayed_search (void *cls)
-{
-    struct CadetPeer *peer = cls;
-
-    peer->search_delayed = NULL;
-    GCC_check_connections ();
-    GCP_start_search (peer);
-    GCC_check_connections ();
-}
+mqm_send_done (void *cls);
 
 
 /**
- * Returns if peer is used (has a tunnel or is neighbor).
+ * Transmit current envelope from this @a mqm.
  *
- * @param peer Peer to check.
- * @return #GNUNET_YES if peer is in use.
+ * @param mqm mqm to transmit message for now
  */
-static int
-peer_is_used (struct CadetPeer *peer)
-{
-    struct CadetPeerPath *p;
-
-    if (NULL != peer->tunnel)
-        return GNUNET_YES;
-
-    for (p = peer->path_head; NULL != p; p = p->next)
-    {
-        if (p->length < 3)
-            return GNUNET_YES;
-    }
-    return GNUNET_NO;
-}
-
-
-/**
- * Iterator over all the peers to get the oldest timestamp.
- *
- * @param cls Closure (unsued).
- * @param key ID of the peer.
- * @param value Peer_Info of the peer.
- */
-static int
-peer_get_oldest (void *cls,
-                 const struct GNUNET_PeerIdentity *key,
-                 void *value)
+static void
+mqm_execute (struct GCP_MessageQueueManager *mqm)
 {
-    struct CadetPeer *p = value;
-    struct GNUNET_TIME_Absolute *abs = cls;
-
-    /* Don't count active peers */
-    if (GNUNET_YES == peer_is_used (p))
-        return GNUNET_YES;
+  struct CadetPeer *cp = mqm->cp;
 
-    if (abs->abs_value_us < p->last_contact.abs_value_us)
-        abs->abs_value_us = p->last_contact.abs_value_us;
-
-    return GNUNET_YES;
+  /* Move ready pointer to the next entry that might be ready. */
+  if ( (mqm == cp->mqm_ready_ptr) &&
+       (NULL != mqm->next) )
+    cp->mqm_ready_ptr = mqm->next;
+  /* Move entry to the end of the DLL, to be fair. */
+  if (mqm != cp->mqm_tail)
+  {
+    GNUNET_CONTAINER_DLL_remove (cp->mqm_head,
+                                 cp->mqm_tail,
+                                 mqm);
+    GNUNET_CONTAINER_DLL_insert_tail (cp->mqm_head,
+                                      cp->mqm_tail,
+                                      mqm);
+  }
+  cp->mqm_ready_counter--;
+  if (GNUNET_YES == should_I_drop ())
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "DROPPING message to peer %s from MQM %p\n",
+         GCP_2s (cp),
+         mqm);
+    GNUNET_MQ_discard (mqm->env);
+    mqm->env = NULL;
+    mqm_send_done (cp);
+  }
+  else
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Sending to peer %s from MQM %p\n",
+         GCP_2s (cp),
+         mqm);
+    GNUNET_MQ_send (cp->core_mq,
+                    mqm->env);
+    mqm->env = NULL;
+  }
+  mqm->cb (mqm->cb_cls,
+           GNUNET_YES);
 }
 
 
 /**
- * Iterator over all the peers to remove the oldest entry.
+ * Find the next ready message in the queue (starting
+ * the search from the `cp->mqm_ready_ptr`) and if possible
+ * execute the transmission.
  *
- * @param cls Closure (unsued).
- * @param key ID of the peer.
- * @param value Peer_Info of the peer.
- */
-static int
-peer_timeout (void *cls,
-              const struct GNUNET_PeerIdentity *key,
-              void *value)
-{
-    struct CadetPeer *p = value;
-    struct GNUNET_TIME_Absolute *abs = cls;
-
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "peer %s timeout\n", GNUNET_i2s (key));
-
-    if (p->last_contact.abs_value_us == abs->abs_value_us &&
-            GNUNET_NO == peer_is_used (p))
-    {
-        peer_destroy (p);
-        return GNUNET_NO;
-    }
-    return GNUNET_YES;
-}
-
-
-/**
- * Delete oldest unused peer.
+ * @param cp peer to try to send the next ready message to
  */
 static void
-peer_delete_oldest (void)
+send_next_ready (struct CadetPeer *cp)
 {
-    struct GNUNET_TIME_Absolute abs;
-
-    abs = GNUNET_TIME_UNIT_FOREVER_ABS;
-
-    GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                           &peer_get_oldest,
-                                           &abs);
-    GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                           &peer_timeout,
-                                           &abs);
-}
-
+  struct GCP_MessageQueueManager *mqm;
 
-/**
- * Choose the best (yet unused) path towards a peer,
- * considering the tunnel properties.
- *
- * @param peer The destination peer.
- * @return Best current known path towards the peer, if any.
- */
-static struct CadetPeerPath *
-peer_get_best_path (const struct CadetPeer *peer)
-{
-    struct CadetPeerPath *best_p;
-    struct CadetPeerPath *p;
-    unsigned int best_cost;
-    unsigned int cost;
-
-    best_cost = UINT_MAX;
-    best_p = NULL;
-    for (p = peer->path_head; NULL != p; p = p->next)
-    {
-        if (GNUNET_NO == path_is_valid (p))
-            continue; /* Don't use invalid paths. */
-        if (GNUNET_YES == GCT_is_path_used (peer->tunnel, p))
-            continue; /* If path is already in use, skip it. */
-
-        if ((cost = GCT_get_path_cost (peer->tunnel, p)) < best_cost)
-        {
-            best_cost = cost;
-            best_p = p;
-        }
-    }
-    return best_p;
+  if (0 == cp->mqm_ready_counter)
+    return;
+  while ( (NULL != (mqm = cp->mqm_ready_ptr)) &&
+          (NULL == mqm->env) )
+    cp->mqm_ready_ptr = mqm->next;
+  if (NULL == mqm)
+    return; /* nothing to do */
+  mqm_execute (mqm);
 }
 
 
 /**
- * Function to process paths received for a new peer addition. The recorded
- * paths form the initial tunnel, which can be optimized later.
- * Called on each result obtained for the DHT search.
+ * Function called when CORE took one of the messages from
+ * a message queue manager and transmitted it.
  *
- * @param cls Closure (peer towards a path has been found).
- * @param path Path created from the DHT query. Will be freed afterwards.
+ * @param cls the `struct CadetPeeer` where we made progress
  */
 static void
-search_handler (void *cls, const struct CadetPeerPath *path)
+mqm_send_done (void *cls)
 {
-    struct CadetPeer *peer = cls;
-    unsigned int connection_count;
-
-    GCC_check_connections ();
-    GCP_add_path_to_all (path, GNUNET_NO);
-
-    /* Count connections */
-    connection_count = GCT_count_connections (peer->tunnel);
+  struct CadetPeer *cp = cls;
 
-    /* If we already have our minimum (or more) connections, it's enough */
-    if (CONNECTIONS_PER_TUNNEL <= connection_count)
-    {
-        GCC_check_connections ();
-        return;
-    }
-
-    if (CADET_TUNNEL_SEARCHING == GCT_get_cstate (peer->tunnel))
-    {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, " ... connect!\n");
-        GCP_connect (peer);
-    }
-    GCC_check_connections ();
-}
-
-
-/**
- * Test if a message type is connection management traffic
- * or regular payload traffic.
- *
- * @param type Message type.
- *
- * @return #GNUNET_YES if connection management, #GNUNET_NO otherwise.
- */
-static int
-is_connection_management (uint16_t type)
-{
-    return type == GNUNET_MESSAGE_TYPE_CADET_CONNECTION_HOP_BY_HOP_ENCRYPTED_ACK ||
-           type == GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED_POLL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending to peer %s completed\n",
+       GCP_2s (cp));
+  send_next_ready (cp);
 }
 
 
 /**
- * Debug function should NEVER return true in production code, useful to
- * simulate losses for testcases.
+ * Send the message in @a env to @a cp.
  *
- * @return #GNUNET_YES or #GNUNET_NO with the decision to drop.
+ * @param mqm the message queue manager to use for transmission
+ * @param env envelope with the message to send; must NOT
+ *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
  */
-static int
-should_I_drop (void)
+void
+GCP_send (struct GCP_MessageQueueManager *mqm,
+          struct GNUNET_MQ_Envelope *env)
 {
-    if (0 == drop_percent)
-        return GNUNET_NO;
+  struct CadetPeer *cp = mqm->cp;
 
-    if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 101) < drop_percent)
-        return GNUNET_YES;
-
-    return GNUNET_NO;
+  GNUNET_assert (NULL != env);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Queueing message to peer %s in MQM %p\n",
+       GCP_2s (cp),
+       mqm);
+  GNUNET_assert (NULL != cp->core_mq);
+  GNUNET_assert (NULL == mqm->env);
+  GNUNET_MQ_notify_sent (env,
+                         &mqm_send_done,
+                         cp);
+  mqm->env = env;
+  cp->mqm_ready_counter++;
+  if (mqm != cp->mqm_ready_ptr)
+    cp->mqm_ready_ptr = cp->mqm_head;
+  if (1 == cp->mqm_ready_counter)
+    cp->mqm_ready_ptr = mqm;
+  if (0 != GNUNET_MQ_get_length (cp->core_mq))
+    return;
+  send_next_ready (cp);
 }
 
 
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
 /**
- * Call the continuation after a message has been sent or dropped.
+ * Function called to destroy a peer now.
  *
- * This funcion removes the message from the queue.
- *
- * @param q Queue handle.
- * @param sent #GNUNET_YES if was sent to CORE, #GNUNET_NO if dropped.
+ * @param cls NULL
+ * @param pid identity of the peer (unused)
+ * @param value the `struct CadetPeer` to clean up
+ * @return #GNUNET_OK (continue to iterate)
  */
-static void
-call_peer_cont (struct CadetPeerQueue *q, int sent)
-{
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " core mq just sent %s\n", GC_m2s (q->type));
-    if (NULL != q->cont)
-    {
-        struct GNUNET_TIME_Relative wait_time;
-
-        wait_time = GNUNET_TIME_absolute_get_duration (q->queue_timestamp);
-        LOG (GNUNET_ERROR_TYPE_DEBUG,
-             " calling callback on %s after %s\n",
-             GCC_2s (q->c),
-             GNUNET_STRINGS_relative_time_to_string (wait_time, GNUNET_NO));
-        q->cont (q->cont_cls,
-                 q->c, q->c_fwd, sent,
-                 q->type,
-                 q->payload_type,
-                 q->payload_id,
-                 q->size, wait_time);
-        q->cont = NULL;
-    }
-    GNUNET_CONTAINER_DLL_remove (q->peer->q_head, q->peer->q_tail, q);
-}
-
-
-/**
- * Function called by MQ when a message is sent to CORE.
- *
- * @param cls Closure (queue handle).
- */
-static void
-mq_sent (void *cls)
+static int
+destroy_iterator_cb (void *cls,
+                     const struct GNUNET_PeerIdentity *pid,
+                     void *value)
 {
-    struct CadetPeerQueue *q = cls;
+  struct CadetPeer *cp = value;
 
-    if (GNUNET_NO == q->management_traffic)
-    {
-        q->peer->queue_n--;
-    }
-    call_peer_cont (q, GNUNET_YES);
-    GNUNET_free (q);
+  if (NULL != cp->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
+  destroy_peer (cp);
+  return GNUNET_OK;
 }
 
 
 /**
- * Finish the drop operation.
- *
- * @param cls queue entry to finish drop for
+ * Clean up all entries about all peers.
+ * Must only be called after all tunnels, CORE-connections and
+ * connections are down.
  */
-static void
-drop_cb (void *cls)
+void
+GCP_destroy_all_peers ()
 {
-  struct CadetPeerQueue *q = cls;
-
-  GNUNET_MQ_discard (q->env);
-  call_peer_cont (q, GNUNET_YES);
-  GNUNET_free (q);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying all peers now\n");
+  GNUNET_CONTAINER_multipeermap_iterate (peers,
+                                         &destroy_iterator_cb,
+                                         NULL);
 }
 
 
 /**
- * @brief Send a message to another peer (using CORE).
+ * Drop all paths owned by this peer, and do not
+ * allow new ones to be added: We are shutting down.
  *
- * @param peer Peer towards which to queue the message.
- * @param message Message to send.
- * @param payload_type Type of the message's payload, for debug messages.
- *                     0 if the message is a retransmission (unknown payload).
- *                     UINT16_MAX if the message does not have payload.
- * @param payload_id ID of the payload (MID, ACK #, etc)
- * @param c Connection this message belongs to (can be NULL).
- * @param fwd Is this a message going root->dest? (FWD ACK are NOT FWD!)
- * @param cont Continuation to be called once CORE has sent the message.
- * @param cont_cls Closure for @c cont.
- *
- * @return A handle to the message in the queue or NULL (if dropped).
+ * @param cp peer to drop paths to
  */
-struct CadetPeerQueue *
-GCP_send (struct CadetPeer *peer,
-          const struct GNUNET_MessageHeader *message,
-          uint16_t payload_type,
-          struct CadetEncryptedMessageIdentifier payload_id,
-          struct CadetConnection *c,
-          int fwd,
-          GCP_sent cont,
-          void *cont_cls)
+void
+GCP_drop_owned_paths (struct CadetPeer *cp)
 {
-    struct CadetPeerQueue *q;
-    uint16_t type;
-    uint16_t size;
-
-    GCC_check_connections ();
-    type = ntohs (message->type);
-    size = ntohs (message->size);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "que %s (%s %4u) on conn %s (%p) %s towards %s (size %u)\n",
-         GC_m2s (type), GC_m2s (payload_type),
-         ntohl (payload_id.pid),
-         GCC_2s (c), c, GC_f2s (fwd), GCP_2s (peer), size);
-
-    if (NULL == peer->connections)
-    {
-        /* We are not connected to this peer, ignore request. */
-        GNUNET_break (0);
-        LOG (GNUNET_ERROR_TYPE_INFO, "%s not a neighbor\n", GCP_2s (peer));
-        GNUNET_STATISTICS_update (stats, "# messages dropped due to wrong hop", 1,
-                                  GNUNET_NO);
-        return NULL;
-    }
+  struct CadetPeerPath *path;
 
-    q = GNUNET_new (struct CadetPeerQueue);
-    q->env = GNUNET_MQ_msg_copy (message);
-    q->peer = peer;
-    q->cont = cont;
-    q->cont_cls = cont_cls;
-    q->queue_timestamp = GNUNET_TIME_absolute_get ();
-    q->management_traffic = is_connection_management (type);
-    q->type = type;
-    q->size = size;
-    q->payload_type = payload_type;
-    q->payload_id = payload_id;
-    q->c = c;
-    q->c_fwd = fwd;
-    GNUNET_MQ_notify_sent (q->env, &mq_sent, q);
-    GNUNET_CONTAINER_DLL_insert (peer->q_head, peer->q_tail, q);
-
-    if (GNUNET_YES == q->management_traffic)
-    {
-        GNUNET_MQ_send (peer->core_mq, q->env);  // FIXME implement "_urgent", use
-    }
-    else
-    {
-        if (GNUNET_YES == should_I_drop ())
-        {
-            LOG (GNUNET_ERROR_TYPE_WARNING,
-                 "DD %s (%s %u) on conn %s %s (random drop for testing)\n",
-                 GC_m2s (q->type),
-                 GC_m2s (q->payload_type),
-                 ntohl (q->payload_id.pid),
-                 GCC_2s (c),
-                 GC_f2s (q->c_fwd));
-            q->drop_task = GNUNET_SCHEDULER_add_now (&drop_cb,
-                                                     q);
-            return q;
-        }
-        GNUNET_MQ_send (peer->core_mq, q->env);
-        peer->queue_n++;
-    }
-
-    GCC_check_connections ();
-    return q;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying all paths to %s\n",
+       GCP_2s (cp));
+  while (NULL != (path =
+                  GNUNET_CONTAINER_heap_remove_root (cp->path_heap)))
+    GCPP_release (path);
+  GNUNET_CONTAINER_heap_destroy (cp->path_heap);
+  cp->path_heap = NULL;
 }
 
 
 /**
- * Cancel sending a message. Message must have been sent with
- * #GCP_send before.  May not be called after the notify sent
- * callback has been called.
- *
- * It DOES call the continuation given to #GCP_send.
+ * Add an entry to the DLL of all of the paths that this peer is on.
  *
- * @param q Queue handle to cancel
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
  */
 void
-GCP_send_cancel (struct CadetPeerQueue *q)
-{
-  if (NULL != q->drop_task)
+GCP_path_entry_add (struct CadetPeer *cp,
+                    struct CadetPeerPathEntry *entry,
+                    unsigned int off)
+{
+  GNUNET_assert (cp == GCPP_get_peer_at_offset (entry->path,
+                                                off));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Discovered that peer %s is on path %s at offset %u\n",
+       GCP_2s (cp),
+       GCPP_2s (entry->path),
+       off);
+  if (off >= cp->path_dll_length)
   {
-    GNUNET_SCHEDULER_cancel (q->drop_task);
-    q->drop_task = NULL;
-    GNUNET_MQ_discard (q->env);
+    unsigned int len = cp->path_dll_length;
+
+    GNUNET_array_grow (cp->path_heads,
+                       len,
+                       off + 4);
+    GNUNET_array_grow (cp->path_tails,
+                       cp->path_dll_length,
+                       off + 4);
   }
-  else
+  GNUNET_CONTAINER_DLL_insert (cp->path_heads[off],
+                               cp->path_tails[off],
+                               entry);
+  cp->off_sum += off;
+  cp->num_paths++;
+
+  /* If we have a tunnel to this peer, tell the tunnel that there is a
+     new path available. */
+  if (NULL != cp->t)
+    GCT_consider_path (cp->t,
+                       entry->path,
+                       off);
+
+  if ( (NULL != cp->search_h) &&
+       (DESIRED_CONNECTIONS_PER_TUNNEL <= cp->num_paths) )
   {
-    GNUNET_MQ_send_cancel (q->env);
+    /* Now I have enough paths, stop search */
+    GCD_search_stop (cp->search_h);
+    cp->search_h = NULL;
+  }
+  if (NULL != cp->destroy_task)
+  {
+    /* paths changed, this resets the destroy timeout counter
+       and aborts a destroy task that may no longer be valid
+       to have (as we now have more paths via this peer). */
+    consider_peer_destroy (cp);
   }
-  call_peer_cont (q, GNUNET_NO);
-  GNUNET_free (q);
 }
 
 
 /**
- * Initialize the peer subsystem.
+ * Remove an entry from the DLL of all of the paths that this peer is on.
  *
- * @param c Configuration.
- */
-void
-GCP_init (const struct GNUNET_CONFIGURATION_Handle *c)
-{
-    cfg = c;
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "GCP_init\n");
-    in_shutdown = GNUNET_NO;
-    peers = GNUNET_CONTAINER_multipeermap_create (128, GNUNET_NO);
-    if (GNUNET_OK !=
-            GNUNET_CONFIGURATION_get_value_number (c, "CADET", "MAX_PEERS",
-                    &max_peers))
-    {
-        GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                                   "CADET", "MAX_PEERS", "USING DEFAULT");
-        max_peers = 1000;
-    }
-
-    if (GNUNET_OK !=
-            GNUNET_CONFIGURATION_get_value_number (c, "CADET", "DROP_PERCENT",
-                    &drop_percent))
-    {
-        drop_percent = 0;
-    }
-    else
-    {
-        LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
-        LOG (GNUNET_ERROR_TYPE_WARNING, "Cadet is running with DROP enabled.\n");
-        LOG (GNUNET_ERROR_TYPE_WARNING, "This is NOT a good idea!\n");
-        LOG (GNUNET_ERROR_TYPE_WARNING, "Remove DROP_PERCENT from config file.\n");
-        LOG (GNUNET_ERROR_TYPE_WARNING, "**************************************\n");
-    }
-    ats_ch = GNUNET_ATS_connectivity_init (c);
-    connect_to_core (c);
-    if (NULL == core_handle)
-    {
-        GNUNET_break (0);
-        GNUNET_SCHEDULER_shutdown ();
-    }
-}
-
-
-/**
- * Shut down the peer subsystem.
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
  */
 void
-GCP_shutdown (void)
-{
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Shutting down peer subsystem\n");
-    in_shutdown = GNUNET_YES;
-    if (NULL != core_handle)
-    {
-        GNUNET_CORE_disconnect (core_handle);
-        core_handle = NULL;
-    }
-    GNUNET_PEER_change_rc (myid, -1);
-    /* With MQ API, CORE calls the disconnect handler for every peer
-     * after calling GNUNET_CORE_disconnect, shutdown must occur *after* that.
-     */
-    GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                           &shutdown_peer,
-                                           NULL);
-    if (NULL != ats_ch)
-    {
-        GNUNET_ATS_connectivity_done (ats_ch);
-        ats_ch = NULL;
-    }
-    GNUNET_CONTAINER_multipeermap_destroy (peers);
-    peers = NULL;
-}
-
-
-/**
- * Retrieve the CadetPeer stucture associated with the peer. Optionally create
- * one and insert it in the appropriate structures if the peer is not known yet.
- *
- * @param peer_id Full identity of the peer.
- * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO otherwise.
- *
- * @return Existing or newly created peer structure.
- *         NULL if unknown and not requested @a create
- */
-struct CadetPeer *
-GCP_get (const struct GNUNET_PeerIdentity *peer_id, int create)
-{
-    struct CadetPeer *peer;
-
-    peer = GNUNET_CONTAINER_multipeermap_get (peers, peer_id);
-    if (NULL == peer)
-    {
-        peer = GNUNET_new (struct CadetPeer);
-        if (GNUNET_CONTAINER_multipeermap_size (peers) > max_peers)
-        {
-            peer_delete_oldest ();
-        }
-        GNUNET_assert (GNUNET_OK ==
-                       GNUNET_CONTAINER_multipeermap_put (peers,
-                               peer_id,
-                               peer,
-                               GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-        peer->id = GNUNET_PEER_intern (peer_id);
-    }
-    peer->last_contact = GNUNET_TIME_absolute_get ();
-
-    return peer;
-}
-
-
-/**
- * Retrieve the CadetPeer stucture associated with the
- * peer. Optionally create one and insert it in the appropriate
- * structures if the peer is not known yet.
- *
- * @param peer Short identity of the peer.
- * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO otherwise.
- *
- * @return Existing or newly created peer structure.
- *         NULL if unknown and not requested @a create
- */
-struct CadetPeer *
-GCP_get_short (const GNUNET_PEER_Id peer, int create)
-{
-    return GCP_get (GNUNET_PEER_resolve2 (peer), create);
+GCP_path_entry_remove (struct CadetPeer *cp,
+                       struct CadetPeerPathEntry *entry,
+                       unsigned int off)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Removing knowledge about peer %s beging on path %s at offset %u\n",
+       GCP_2s (cp),
+       GCPP_2s (entry->path),
+       off);
+  GNUNET_CONTAINER_DLL_remove (cp->path_heads[off],
+                               cp->path_tails[off],
+                               entry);
+  GNUNET_assert (0 < cp->num_paths);
+  cp->off_sum -= off;
+  cp->num_paths--;
+  if ( (NULL == cp->core_mq) &&
+       (NULL != cp->t) &&
+       (NULL == cp->search_h) &&
+       (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) )
+    cp->search_h
+      = GCD_search (&cp->pid);
+  if (NULL == cp->destroy_task)
+  {
+    /* paths changed, we might now be ready for destruction, check again */
+    consider_peer_destroy (cp);
+  }
 }
 
 
 /**
- * Function called once #GNUNET_TRANSPORT_offer_hello() is done.
- * Marks the operation as finished.
+ * Prune down the number of paths to this peer, we seem to
+ * have way too many.
  *
- * @param cls Closure (our `struct CadetPeer`).
+ * @param cls the `struct CadetPeer` to maintain the path heap for
  */
 static void
-hello_offer_done (void *cls)
+path_heap_cleanup (void *cls)
 {
-    struct CadetPeer *peer = cls;
+  struct CadetPeer *cp = cls;
+  struct CadetPeerPath *root;
 
-    peer->hello_offer = NULL;
+  cp->heap_cleanup_task = NULL;
+  while (GNUNET_CONTAINER_heap_get_size (cp->path_heap) >=
+         2 * DESIRED_CONNECTIONS_PER_TUNNEL)
+  {
+    /* Now we have way too many, drop least desirable UNLESS it is in use!
+       (Note that this intentionally keeps highly desireable, but currently
+       unused paths around in the hope that we might be able to switch, even
+       if the number of paths exceeds the threshold.) */
+    root = GNUNET_CONTAINER_heap_peek (cp->path_heap);
+    GNUNET_assert (NULL != root);
+    if (NULL !=
+        GCPP_get_connection (root,
+                             cp,
+                             GCPP_get_length (root) - 1))
+      break; /* can't fix */
+    /* Got plenty of paths to this destination, and this is a low-quality
+       one that we don't care about. Allow it to die. */
+    GNUNET_assert (root ==
+                   GNUNET_CONTAINER_heap_remove_root (cp->path_heap));
+    GCPP_release (root);
+  }
 }
 
 
 /**
- * Try to establish a new connection to this peer (in its tunnel).
- * If the peer doesn't have any path to it yet, try to get one.
- * If the peer already has some path, send a CREATE CONNECTION towards it.
+ * Try adding a @a path to this @a peer.  If the peer already
+ * has plenty of paths, return NULL.
  *
- * @param peer Peer to connect to.
+ * @param cp peer to which the @a path leads to
+ * @param path a path looking for an owner; may not be fully initialized yet!
+ * @param off offset of @a cp in @a path
+ * @param force force attaching the path
+ * @return NULL if this peer does not care to become a new owner,
+ *         otherwise the node in the peer's path heap for the @a path.
  */
-void
-GCP_connect (struct CadetPeer *peer)
+struct GNUNET_CONTAINER_HeapNode *
+GCP_attach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 unsigned int off,
+                 int force)
 {
-    struct CadetTunnel *t;
-    struct CadetPeerPath *path;
-    struct CadetConnection *c;
-    int rerun_search;
-
-    GCC_check_connections ();
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "peer_connect towards %s\n",
-         GCP_2s (peer));
-    /* If we have a current hello, try to connect using it. */
-    GCP_try_connect (peer);
+  GNUNET_CONTAINER_HeapCostType desirability;
+  struct CadetPeerPath *root;
+  GNUNET_CONTAINER_HeapCostType root_desirability;
+  struct GNUNET_CONTAINER_HeapNode *hn;
 
-    t = peer->tunnel;
-    c = NULL;
-    rerun_search = GNUNET_NO;
-
-    if (NULL != peer->path_head)
+  GNUNET_assert (off == GCPP_get_length (path) - 1);
+  GNUNET_assert (cp == GCPP_get_peer_at_offset (path,
+                                                off));
+  if (NULL == cp->path_heap)
+  {
+    /* #GCP_drop_owned_paths() was already called, we cannot take new ones! */
+    GNUNET_assert (GNUNET_NO == force);
+    return NULL;
+  }
+  desirability = GCPP_get_desirability (path);
+  if (GNUNET_NO == force)
+  {
+    /* FIXME: desirability is not yet initialized; tricky! */
+    if (GNUNET_NO ==
+        GNUNET_CONTAINER_heap_peek2 (cp->path_heap,
+                                     (void **) &root,
+                                     &root_desirability))
     {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "  some path exists\n");
-        path = peer_get_best_path (peer);
-        if (NULL != path)
-        {
-            char *s;
-
-            s = path_2s (path);
-            LOG (GNUNET_ERROR_TYPE_DEBUG, "  path to use: %s\n", s);
-            GNUNET_free (s);
-
-            c = GCT_use_path (t, path);
-            if (NULL == c)
-            {
-                /* This case can happen when the path includes a first hop that is
-                 * not yet known to be connected.
-                 *
-                 * This happens quite often during testing when running cadet
-                 * under valgrind: core connect notifications come very late
-                 * and the DHT result has already come and created a valid
-                 * path.  In this case, the peer->connections
-                 * hashmaps will be NULL and tunnel_use_path will not be able
-                 * to create a connection from that path.
-                 *
-                 * Re-running the DHT GET should give core time to callback.
-                 *
-                 * GCT_use_path -> GCC_new -> register_neighbors takes care of
-                 * updating statistics about this issue.
-                 */
-                rerun_search = GNUNET_YES;
-            }
-            else
-            {
-                GCC_send_create (c);
-                return;
-            }
-        }
-        else
-        {
-            LOG (GNUNET_ERROR_TYPE_DEBUG, "  but is NULL, all paths are in use\n");
-        }
+      root = NULL;
+      root_desirability = 0;
     }
 
-    if (GNUNET_YES == rerun_search)
+    if ( (DESIRED_CONNECTIONS_PER_TUNNEL > cp->num_paths) &&
+         (desirability < root_desirability) )
     {
-        struct GNUNET_TIME_Relative delay;
-
-        GCP_stop_search (peer);
-        delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 100);
-        peer->search_delayed = GNUNET_SCHEDULER_add_delayed (delay,
-                               &delayed_search,
-                               peer);
-        GCC_check_connections ();
-        return;
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Decided to not attach path %p to peer %s due to undesirability\n",
+           GCPP_2s (path),
+           GCP_2s (cp));
+      return NULL;
     }
+  }
 
-    if (GNUNET_NO == is_searching (peer))
-        GCP_start_search (peer);
-    GCC_check_connections ();
-}
-
-
-/**
- * Chech whether there is a direct (core level)  connection to peer.
- *
- * @param peer Peer to check.
- *
- * @return #GNUNET_YES if there is a direct connection.
- */
-int
-GCP_is_neighbor (const struct CadetPeer *peer)
-{
-    struct CadetPeerPath *path;
-
-    if (NULL == peer->connections)
-        return GNUNET_NO;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Attaching path %s to peer %s (%s)\n",
+       GCPP_2s (path),
+       GCP_2s (cp),
+       (GNUNET_NO == force) ? "desirable" : "forced");
 
-    for (path = peer->path_head; NULL != path; path = path->next)
-    {
-        if (3 > path->length)
-            return GNUNET_YES;
-    }
+  /* Yes, we'd like to add this path, add to our heap */
+  hn = GNUNET_CONTAINER_heap_insert (cp->path_heap,
+                                     path,
+                                     desirability);
 
-    /* Is not a neighbor but connections is not NULL, probably disconnecting */
-    return GNUNET_NO;
+  /* Consider maybe dropping other paths because of the new one */
+  if ( (GNUNET_CONTAINER_heap_get_size (cp->path_heap) >=
+        2 * DESIRED_CONNECTIONS_PER_TUNNEL) &&
+       (NULL != cp->heap_cleanup_task) )
+    cp->heap_cleanup_task = GNUNET_SCHEDULER_add_now (&path_heap_cleanup,
+                                                      cp);
+  return hn;
 }
 
 
 /**
- * Create and initialize a new tunnel towards a peer, in case it has none.
- * In case the peer already has a tunnel, nothing is done.
+ * This peer can no longer own @a path as the path
+ * has been extended and a peer further down the line
+ * is now the new owner.
  *
- * Does not generate any traffic, just creates the local data structures.
- *
- * @param peer Peer towards which to create the tunnel.
+ * @param cp old owner of the @a path
+ * @param path path where the ownership is lost
+ * @param hn note in @a cp's path heap that must be deleted
  */
 void
-GCP_add_tunnel (struct CadetPeer *peer)
+GCP_detach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 struct GNUNET_CONTAINER_HeapNode *hn)
 {
-    GCC_check_connections ();
-    if (NULL != peer->tunnel)
-        return;
-    peer->tunnel = GCT_new (peer);
-    GCC_check_connections ();
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Detatching path %s from peer %s\n",
+       GCPP_2s (path),
+       GCP_2s (cp));
+  GNUNET_assert (path ==
+                 GNUNET_CONTAINER_heap_remove_node (hn));
 }
 
 
 /**
- * Add a connection to a neighboring peer.
- *
- * Store that the peer is the first hop of the connection in one
- * direction and that on peer disconnect the connection must be
- * notified and destroyed, for it will no longer be valid.
+ * Add a @a connection to this @a cp.
  *
- * @param peer Peer to add connection to.
- * @param c Connection to add.
- * @param pred #GNUNET_YES if we are predecessor, #GNUNET_NO if we are successor
+ * @param cp peer via which the @a connection goes
+ * @param cc the connection to add
  */
 void
-GCP_add_connection (struct CadetPeer *peer,
-                    struct CadetConnection *c,
-                    int pred)
-{
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "adding connection %s\n",
-         GCC_2s (c));
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "to peer %s\n",
-         GCP_2s (peer));
-    GNUNET_assert (NULL != peer->connections);
-    GNUNET_assert (GNUNET_OK ==
-                   GNUNET_CONTAINER_multishortmap_put (peer->connections,
-                                                       &GCC_get_id (c)->connection_of_tunnel,
-                                                       c,
-                                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Peer %s has now %u connections.\n",
-         GCP_2s (peer),
-         GNUNET_CONTAINER_multishortmap_size (peer->connections));
-}
-
-
-/**
- * Add the path to the peer and update the path used to reach it in case this
- * is the shortest.
- *
- * @param peer Destination peer to add the path to.
- * @param path New path to add. Last peer must be @c peer.
- *             Path will be either used of freed if already known.
- * @param trusted Do we trust that this path is real?
- *
- * @return path if path was taken, pointer to existing duplicate if exists
- *         NULL on error.
- */
-struct CadetPeerPath *
-GCP_add_path (struct CadetPeer *peer,
-              struct CadetPeerPath *path,
-              int trusted)
-{
-    struct CadetPeerPath *aux;
-    unsigned int l;
-    unsigned int l2;
-
-    GCC_check_connections ();
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "adding path [%u] to peer %s\n",
-         path->length, GCP_2s (peer));
-
-    if (NULL == peer || NULL == path
-            || path->peers[path->length - 1] != peer->id)
-    {
-        GNUNET_break (0);
-        path_destroy (path);
-        return NULL;
-    }
-
-    for (l = 1; l < path->length; l++)
-    {
-        if (path->peers[l] == myid)
-        {
-            LOG (GNUNET_ERROR_TYPE_DEBUG, " shortening path by %u\n", l);
-            for (l2 = 0; l2 < path->length - l; l2++)
-            {
-                path->peers[l2] = path->peers[l + l2];
-            }
-            path->length -= l;
-            l = 1;
-            path->peers = GNUNET_realloc (path->peers,
-                                          path->length * sizeof (GNUNET_PEER_Id));
-        }
-    }
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " final length: %u\n", path->length);
-
-    if (2 >= path->length && GNUNET_NO == trusted)
-    {
-        /* Only allow CORE to tell us about direct paths */
-        path_destroy (path);
-        return NULL;
-    }
-
-    l = path_get_length (path);
-    if (0 == l)
-    {
-        path_destroy (path);
-        return NULL;
-    }
-
-    GNUNET_assert (peer->id == path->peers[path->length - 1]);
-    for (aux = peer->path_head; aux != NULL; aux = aux->next)
-    {
-        l2 = path_get_length (aux);
-        if (l2 > l)
-        {
-            LOG (GNUNET_ERROR_TYPE_DEBUG, "  added\n");
-            GNUNET_CONTAINER_DLL_insert_before (peer->path_head,
-                                                peer->path_tail, aux, path);
-            goto finish;
-        }
-        else
-        {
-            if (l2 == l && memcmp (path->peers, aux->peers, l) == 0)
-            {
-                LOG (GNUNET_ERROR_TYPE_DEBUG, "  already known\n");
-                path_destroy (path);
-                return aux;
-            }
-        }
-    }
-    GNUNET_CONTAINER_DLL_insert_tail (peer->path_head,
-                                      peer->path_tail,
-                                      path);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  added last\n");
-
-finish:
-    if (NULL != peer->tunnel
-            && CONNECTIONS_PER_TUNNEL > GCT_count_connections (peer->tunnel)
-            && 2 < path->length) /* Direct paths are handled by core_connect */
-    {
-        GCP_connect (peer);
-    }
-    GCC_check_connections ();
-    return path;
+GCP_add_connection (struct CadetPeer *cp,
+                    struct CadetConnection *cc)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Adding connection %s to peer %s\n",
+       GCC_2s (cc),
+       GCP_2s (cp));
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multishortmap_put (cp->connections,
+                                                     &GCC_get_id (cc)->connection_of_tunnel,
+                                                     cc,
+                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  if (NULL != cp->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (cp->destroy_task);
+    cp->destroy_task = NULL;
+  }
 }
 
 
 /**
- * Add the path to the origin peer and update the path used to reach it in case
- * this is the shortest.
- * The path is given in peer_info -> destination, therefore we turn the path
- * upside down first.
- *
- * @param peer Peer to add the path to, being the origin of the path.
- * @param path New path to add after being inversed.
- *             Path will be either used or freed.
- * @param trusted Do we trust that this path is real?
+ * Remove a @a connection that went via this @a cp.
  *
- * @return path if path was taken, pointer to existing duplicate if exists
- *         NULL on error.
+ * @param cp peer via which the @a connection went
+ * @param cc the connection to remove
  */
-struct CadetPeerPath *
-GCP_add_path_to_origin (struct CadetPeer *peer,
-                        struct CadetPeerPath *path,
-                        int trusted)
+void
+GCP_remove_connection (struct CadetPeer *cp,
+                       struct CadetConnection *cc)
 {
-    if (NULL == path)
-        return NULL;
-    path_invert (path);
-    return GCP_add_path (peer, path, trusted);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Removing connection %s from peer %s\n",
+       GCC_2s (cc),
+       GCP_2s (cp));
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multishortmap_remove (cp->connections,
+                                                        &GCC_get_id (cc)->connection_of_tunnel,
+                                                        cc));
+  consider_peer_destroy (cp);
 }
 
 
 /**
- * Adds a path to the info of all the peers in the path
+ * Retrieve the CadetPeer stucture associated with the
+ * peer. Optionally create one and insert it in the appropriate
+ * structures if the peer is not known yet.
  *
- * @param p Path to process.
- * @param confirmed Whether we know if the path works or not.
+ * @param peer_id Full identity of the peer.
+ * @param create #GNUNET_YES if a new peer should be created if unknown.
+ *               #GNUNET_NO to return NULL if peer is unknown.
+ * @return Existing or newly created peer structure.
+ *         NULL if unknown and not requested @a create
  */
-void
-GCP_add_path_to_all (const struct CadetPeerPath *p, int confirmed)
+struct CadetPeer *
+GCP_get (const struct GNUNET_PeerIdentity *peer_id,
+         int create)
 {
-    unsigned int i;
+  struct CadetPeer *cp;
 
-    /* TODO: invert and add to origin */
-    /* TODO: replace all "GCP_add_path" with this, make the other one static */
-    GCC_check_connections ();
-    for (i = 0; i < p->length && p->peers[i] != myid; i++) /* skip'em */ ;
-    for (i++; i < p->length; i++)
-    {
-        struct CadetPeer *peer;
-        struct CadetPeerPath *copy;
-
-        peer = GCP_get_short (p->peers[i], GNUNET_YES);
-        copy = path_duplicate (p);
-        copy->length = i + 1;
-        GCP_add_path (peer, copy, 3 > p->length ? GNUNET_NO : confirmed);
-    }
-    GCC_check_connections ();
+  cp = GNUNET_CONTAINER_multipeermap_get (peers,
+                                          peer_id);
+  if (NULL != cp)
+    return cp;
+  if (GNUNET_NO == create)
+    return NULL;
+  cp = GNUNET_new (struct CadetPeer);
+  cp->pid = *peer_id;
+  cp->connections = GNUNET_CONTAINER_multishortmap_create (32,
+                                                           GNUNET_YES);
+  cp->path_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multipeermap_put (peers,
+                                                    &cp->pid,
+                                                    cp,
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating peer %s\n",
+       GCP_2s (cp));
+  return cp;
 }
 
 
 /**
- * Remove any path to the peer that has the exact same peers as the one given.
+ * Obtain the peer identity for a `struct CadetPeer`.
  *
- * @param peer Peer to remove the path from.
- * @param path Path to remove. Is always destroyed .
+ * @param cp our peer handle
+ * @return the peer identity
  */
-void
-GCP_remove_path (struct CadetPeer *peer,
-                 struct CadetPeerPath *path)
+const struct GNUNET_PeerIdentity *
+GCP_get_id (struct CadetPeer *cp)
 {
-    struct CadetPeerPath *iter;
-    struct CadetPeerPath *next;
-
-    GCC_check_connections ();
-    GNUNET_assert (myid == path->peers[0]);
-    GNUNET_assert (peer->id == path->peers[path->length - 1]);
-
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         "Removing path %p (%u) from %s\n",
-         path, path->length, GCP_2s (peer));
-
-    for (iter = peer->path_head; NULL != iter; iter = next)
-    {
-        next = iter->next;
-        if (0 == path_cmp (path, iter))
-        {
-            GNUNET_CONTAINER_DLL_remove (peer->path_head,
-                                         peer->path_tail,
-                                         iter);
-            if (iter != path)
-                path_destroy (iter);
-        }
-    }
-    path_destroy (path);
-    GCC_check_connections ();
+  return &cp->pid;
 }
 
 
 /**
- * Check that we are aware of a connection from a neighboring peer.
+ * Iterate over all known peers.
  *
- * @param peer Peer to the connection is with
- * @param c Connection that should be in the map with this peer.
+ * @param iter Iterator.
+ * @param cls Closure for @c iter.
  */
 void
-GCP_check_connection (const struct CadetPeer *peer,
-                      const struct CadetConnection *c)
+GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
+                 void *cls)
 {
-  GNUNET_assert (NULL != peer);
-  GNUNET_assert (NULL != peer->connections);
-  return; // ????
-  GNUNET_assert (GNUNET_YES ==
-                 GNUNET_CONTAINER_multishortmap_contains_value (peer->connections,
-                                                                &GCC_get_id (c)->connection_of_tunnel,
-                                                                c));
+  GNUNET_CONTAINER_multipeermap_iterate (peers,
+                                         iter,
+                                         cls);
 }
 
 
 /**
- * Remove a connection from a neighboring peer.
+ * Count the number of known paths toward the peer.
  *
- * @param peer Peer to remove connection from.
- * @param c Connection to remove.
+ * @param cp Peer to get path info.
+ * @return Number of known paths.
  */
-void
-GCP_remove_connection (struct CadetPeer *peer,
-                       const struct CadetConnection *c)
+unsigned int
+GCP_count_paths (const struct CadetPeer *cp)
 {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Removing connection %s\n",
-         GCC_2s (c));
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "from peer %s\n",
-         GCP_2s (peer));
-    if ( (NULL == peer) ||
-            (NULL == peer->connections) )
-        return;
-    GNUNET_assert (GNUNET_YES ==
-                   GNUNET_CONTAINER_multishortmap_remove (peer->connections,
-                                                          &GCC_get_id (c)->connection_of_tunnel,
-                                                          c));
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Peer %s remains with %u connections.\n",
-         GCP_2s (peer),
-         GNUNET_CONTAINER_multishortmap_size (peer->connections));
+  return cp->num_paths;
 }
 
 
 /**
- * Start the DHT search for new paths towards the peer: we don't have
- * enough good connections.
+ * Iterate over the paths to a peer.
  *
- * @param peer Destination peer.
+ * @param cp Peer to get path info.
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
  */
-void
-GCP_start_search (struct CadetPeer *peer)
+unsigned int
+GCP_iterate_paths (struct CadetPeer *cp,
+                   GCP_PathIterator callback,
+                   void *callback_cls)
 {
-    const struct GNUNET_PeerIdentity *id;
-    struct CadetTunnel *t = peer->tunnel;
-
-    GCC_check_connections ();
-    if (NULL != peer->search_h)
-    {
-        GNUNET_break (0);
-        return;
-    }
-
-    if (NULL != peer->search_delayed)
-        GCP_stop_search (peer);
-
-    id = GNUNET_PEER_resolve2 (peer->id);
-    peer->search_h = GCD_search (id, &search_handler, peer);
-
-    if (NULL == t)
-    {
-        /* Why would we search for a peer with no tunnel towards it? */
-        GNUNET_break (0);
-        return;
-    }
-
-    if (CADET_TUNNEL_NEW == GCT_get_cstate (t)
-            || 0 == GCT_count_any_connections (t))
-    {
-        GCT_change_cstate (t, CADET_TUNNEL_SEARCHING);
-    }
-    GCC_check_connections ();
-}
+  unsigned int ret = 0;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Iterating over paths to peer %s%s\n",
+       GCP_2s (cp),
+       (NULL == cp->core_mq) ? "" : " including direct link");
+  if (NULL != cp->core_mq)
+  {
+    struct CadetPeerPath *path;
 
-/**
- * Stop the DHT search for new paths towards the peer: we already have
- * enough good connections.
- *
- * @param peer Destination peer.
- */
-void
-GCP_stop_search (struct CadetPeer *peer)
-{
-    GCC_check_connections ();
-    if (NULL != peer->search_h)
-    {
-        GCD_search_stop (peer->search_h);
-        peer->search_h = NULL;
-    }
-    if (NULL != peer->search_delayed)
+    path = GCPP_get_path_from_route (1,
+                                     &cp->pid);
+    ret++;
+    if (GNUNET_NO ==
+        callback (callback_cls,
+                  path,
+                  0))
+      return ret;
+  }
+  for (unsigned int i=0;i<cp->path_dll_length;i++)
+  {
+    for (struct CadetPeerPathEntry *pe = cp->path_heads[i];
+         NULL != pe;
+         pe = pe->next)
     {
-        GNUNET_SCHEDULER_cancel (peer->search_delayed);
-        peer->search_delayed = NULL;
+      ret++;
+      if (GNUNET_NO ==
+          callback (callback_cls,
+                    pe->path,
+                    i))
+        return ret;
     }
-    GCC_check_connections ();
-}
-
-
-/**
- * Get the Full ID of a peer.
- *
- * @param peer Peer to get from.
- *
- * @return Full ID of peer.
- */
-const struct GNUNET_PeerIdentity *
-GCP_get_id (const struct CadetPeer *peer)
-{
-    return GNUNET_PEER_resolve2 (peer->id);
+  }
+  return ret;
 }
 
 
 /**
- * Get the Short ID of a peer.
- *
- * @param peer Peer to get from.
+ * Iterate over the paths to @a cp where
+ * @a cp is at distance @a dist from us.
  *
- * @return Short ID of peer.
+ * @param cp Peer to get path info.
+ * @param dist desired distance of @a cp to us on the path
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
  */
-GNUNET_PEER_Id
-GCP_get_short_id (const struct CadetPeer *peer)
+unsigned int
+GCP_iterate_paths_at (struct CadetPeer *cp,
+                      unsigned int dist,
+                      GCP_PathIterator callback,
+                      void *callback_cls)
 {
-    return peer->id;
-}
-
+  unsigned int ret = 0;
 
-/**
- * Set tunnel.
- *
- * If tunnel is NULL and there was a search active, stop it, as it's useless.
- *
- * @param peer Peer.
- * @param t Tunnel.
- */
-void
-GCP_set_tunnel (struct CadetPeer *peer, struct CadetTunnel *t)
-{
-    peer->tunnel = t;
-    if (NULL == t && GNUNET_YES == is_searching (peer))
-    {
-        GCP_stop_search (peer);
-    }
+  if (dist >= cp->path_dll_length)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Asked to look for paths at distance %u, but maximum for me is < %u\n",
+         dist,
+         cp->path_dll_length);
+    return 0;
+  }
+  for (struct CadetPeerPathEntry *pe = cp->path_heads[dist];
+       NULL != pe;
+       pe = pe->next)
+  {
+    if (GNUNET_NO ==
+        callback (callback_cls,
+                  pe->path,
+                  dist))
+      return ret;
+    ret++;
+  }
+  return ret;
 }
 
 
 /**
  * Get the tunnel towards a peer.
  *
- * @param peer Peer to get from.
- *
+ * @param cp Peer to get from.
+ * @param create #GNUNET_YES to create a tunnel if we do not have one
  * @return Tunnel towards peer.
  */
 struct CadetTunnel *
-GCP_get_tunnel (const struct CadetPeer *peer)
+GCP_get_tunnel (struct CadetPeer *cp,
+                int create)
 {
-    if (NULL == peer)
-        return NULL;
-    return peer->tunnel;
+  if (NULL == cp)
+    return NULL;
+  if ( (NULL != cp->t) ||
+       (GNUNET_NO == create) )
+    return cp->t;
+  cp->t = GCT_create_tunnel (cp);
+  consider_peer_activate (cp);
+  return cp->t;
 }
 
 
 /**
- * Set the hello message.
+ * Hello offer was passed to the transport service. Mark it
+ * as done.
  *
- * @param peer Peer whose message to set.
- * @param hello Hello message.
+ * @param cls the `struct CadetPeer` where the offer completed
  */
-void
-GCP_set_hello (struct CadetPeer *peer,
-               const struct GNUNET_HELLO_Message *hello)
+static void
+hello_offer_done (void *cls)
 {
-    struct GNUNET_HELLO_Message *old;
-    size_t size;
+  struct CadetPeer *cp = cls;
 
-    GCC_check_connections ();
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "set hello for %s\n", GCP_2s (peer));
-    if (NULL == hello)
-        return;
-
-    old = GCP_get_hello (peer);
-    if (NULL == old)
-    {
-        size = GNUNET_HELLO_size (hello);
-        peer->hello = GNUNET_malloc (size);
-        GNUNET_memcpy (peer->hello, hello, size);
-    }
-    else
-    {
-        peer->hello = GNUNET_HELLO_merge (old, hello);
-        GNUNET_free (old);
-    }
-    GCC_check_connections ();
+  cp->hello_offer = NULL;
 }
 
 
 /**
- * Get the hello message.
- *
- * @param peer Peer whose message to get.
+ * We got a HELLO for a @a peer, remember it, and possibly
+ * trigger adequate actions (like trying to connect).
  *
- * @return Hello message.
+ * @param cp the peer we got a HELLO for
+ * @param hello the HELLO to remember
  */
-struct GNUNET_HELLO_Message *
-GCP_get_hello (struct CadetPeer *peer)
+void
+GCP_set_hello (struct CadetPeer *cp,
+               const struct GNUNET_HELLO_Message *hello)
 {
-    struct GNUNET_TIME_Absolute expiration;
-    struct GNUNET_TIME_Relative remaining;
-
-    if (NULL == peer->hello)
-        return NULL;
+  struct GNUNET_HELLO_Message *mrg;
 
-    expiration = GNUNET_HELLO_get_last_expiration (peer->hello);
-    remaining = GNUNET_TIME_absolute_get_remaining (expiration);
-    if (0 == remaining.rel_value_us)
-    {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, " get - hello expired on %s\n",
-             GNUNET_STRINGS_absolute_time_to_string (expiration));
-        GNUNET_free (peer->hello);
-        peer->hello = NULL;
-    }
-    return peer->hello;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got %u byte HELLO for peer %s\n",
+       (unsigned int) GNUNET_HELLO_size (hello),
+       GCP_2s (cp));
+  if (NULL != cp->hello_offer)
+  {
+    GNUNET_TRANSPORT_offer_hello_cancel (cp->hello_offer);
+    cp->hello_offer = NULL;
+  }
+  if (NULL != cp->hello)
+  {
+    mrg = GNUNET_HELLO_merge (hello,
+                              cp->hello);
+    GNUNET_free (cp->hello);
+    cp->hello = mrg;
+  }
+  else
+  {
+    cp->hello = GNUNET_memdup (hello,
+                               GNUNET_HELLO_size (hello));
+  }
+  cp->hello_offer
+    = GNUNET_TRANSPORT_offer_hello (cfg,
+                                    GNUNET_HELLO_get_header (cp->hello) ,
+                                    &hello_offer_done,
+                                    cp);
+  /* New HELLO means cp's destruction time may change... */
+  consider_peer_destroy (cp);
 }
 
 
 /**
- * Try to connect to a peer on TRANSPORT level.
+ * The tunnel to the given peer no longer exists, remove it from our
+ * data structures, and possibly clean up the peer itself.
  *
- * @param peer Peer to whom to connect.
+ * @param cp the peer affected
+ * @param t the dead tunnel
  */
 void
-GCP_try_connect (struct CadetPeer *peer)
+GCP_drop_tunnel (struct CadetPeer *cp,
+                 struct CadetTunnel *t)
 {
-    struct GNUNET_HELLO_Message *hello;
-    struct GNUNET_MessageHeader *mh;
-
-    if (GNUNET_YES !=
-            GNUNET_CONFIGURATION_get_value_yesno (cfg,
-                    "CADET",
-                    "DISABLE_TRY_CONNECT"))
-        return;
-    GCC_check_connections ();
-    if (GNUNET_YES == GCP_is_neighbor (peer))
-        return;
-    hello = GCP_get_hello (peer);
-    if (NULL == hello)
-        return;
-
-    mh = GNUNET_HELLO_get_header (hello);
-    if (NULL != peer->hello_offer)
-    {
-        GNUNET_TRANSPORT_offer_hello_cancel (peer->hello_offer);
-        peer->hello_offer = NULL;
-    }
-    peer->hello_offer = GNUNET_TRANSPORT_offer_hello (cfg,
-                        mh,
-                        &hello_offer_done,
-                        peer);
-    if (NULL == peer->connectivity_suggestion)
-        peer->connectivity_suggestion
-            = GNUNET_ATS_connectivity_suggest (ats_ch,
-                                               GCP_get_id (peer),
-                                               1);  /* strength */
-    GCC_check_connections ();
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Dropping tunnel %s to peer %s\n",
+       GCT_2s (t),
+       GCP_2s (cp));
+  GNUNET_assert (cp->t == t);
+  cp->t = NULL;
+  consider_peer_destroy (cp);
 }
 
 
 /**
- * Notify a peer that a link between two other peers is broken. If any path
- * used that link, eliminate it.
+ * Test if @a cp has a core-level connection
  *
- * @param peer Peer affected by the change.
- * @param peer1 Peer whose link is broken.
- * @param peer2 Peer whose link is broken.
+ * @param cp peer to test
+ * @return #GNUNET_YES if @a cp has a core-level connection
  */
-void
-GCP_notify_broken_link (struct CadetPeer *peer,
-                        const struct GNUNET_PeerIdentity *peer1,
-                        const struct GNUNET_PeerIdentity *peer2)
+int
+GCP_has_core_connection (struct CadetPeer *cp)
 {
-    struct CadetPeerPath *iter;
-    struct CadetPeerPath *next;
-    unsigned int i;
-    GNUNET_PEER_Id p1;
-    GNUNET_PEER_Id p2;
-
-    GCC_check_connections ();
-    p1 = GNUNET_PEER_search (peer1);
-    p2 = GNUNET_PEER_search (peer2);
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Link %u-%u broken\n", p1, p2);
-    if (0 == p1 || 0 == p2)
-    {
-        /* We don't even know them */
-        return;
-    }
-
-    for (iter = peer->path_head; NULL != iter; iter = next)
-    {
-        next = iter->next;
-        for (i = 0; i < iter->length - 1; i++)
-        {
-            if ((iter->peers[i] == p1 && iter->peers[i + 1] == p2)
-                    || (iter->peers[i] == p2 && iter->peers[i + 1] == p1))
-            {
-                char *s;
-
-                s = path_2s (iter);
-                LOG (GNUNET_ERROR_TYPE_DEBUG, " - invalidating %s\n", s);
-                GNUNET_free (s);
-
-                path_invalidate (iter);
-            }
-        }
-    }
-    GCC_check_connections ();
+  return (NULL != cp->core_mq) ? GNUNET_YES : GNUNET_NO;
 }
 
 
 /**
- * Count the number of known paths toward the peer.
- *
- * @param peer Peer to get path info.
+ * Start message queue change notifications.
  *
- * @return Number of known paths.
+ * @param cp peer to notify for
+ * @param cb function to call if mq becomes available or unavailable
+ * @param cb_cls closure for @a cb
+ * @return handle to cancel request
  */
-unsigned int
-GCP_count_paths (const struct CadetPeer *peer)
+struct GCP_MessageQueueManager *
+GCP_request_mq (struct CadetPeer *cp,
+                GCP_MessageQueueNotificationCallback cb,
+                void *cb_cls)
 {
-    struct CadetPeerPath *iter;
-    unsigned int i;
+  struct GCP_MessageQueueManager *mqm;
 
-    for (iter = peer->path_head, i = 0; NULL != iter; iter = iter->next)
-        i++;
-
-    return i;
+  mqm = GNUNET_new (struct GCP_MessageQueueManager);
+  mqm->cb = cb;
+  mqm->cb_cls = cb_cls;
+  mqm->cp = cp;
+  GNUNET_CONTAINER_DLL_insert (cp->mqm_head,
+                               cp->mqm_tail,
+                               mqm);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating MQM %p for peer %s\n",
+       mqm,
+       GCP_2s (cp));
+  if (NULL != cp->core_mq)
+    cb (cb_cls,
+        GNUNET_YES);
+  return mqm;
 }
 
 
 /**
- * Iterate over the paths to a peer.
+ * Stops message queue change notifications.
  *
- * @param peer Peer to get path info.
- * @param callback Function to call for every path.
- * @param cls Closure for @a callback.
- *
- * @return Number of iterated paths.
+ * @param mqm handle matching request to cancel
+ * @param last_env final message to transmit, or NULL
  */
-unsigned int
-GCP_iterate_paths (struct CadetPeer *peer,
-                   GCP_path_iterator callback,
-                   void *cls)
-{
-    struct CadetPeerPath *iter;
-    unsigned int i;
-
-    for (iter = peer->path_head, i = 0; NULL != iter; iter = iter->next)
+void
+GCP_request_mq_cancel (struct GCP_MessageQueueManager *mqm,
+                       struct GNUNET_MQ_Envelope *last_env)
+{
+  struct CadetPeer *cp = mqm->cp;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying MQM %p for peer %s%s\n",
+       mqm,
+       GCP_2s (cp),
+       (NULL == last_env) ? "" : " with last ditch transmission");
+  if (NULL != mqm->env)
+    GNUNET_MQ_discard (mqm->env);
+  if (NULL != last_env)
+  {
+    if (NULL != cp->core_mq)
     {
-        i++;
-        if (GNUNET_YES != callback (cls, peer, iter))
-            break;
+      GNUNET_MQ_notify_sent (last_env,
+                             &mqm_send_done,
+                             cp);
+      GNUNET_MQ_send (cp->core_mq,
+                      last_env);
     }
-
-    return i;
+    else
+    {
+      GNUNET_MQ_discard (last_env);
+    }
+  }
+  if (cp->mqm_ready_ptr == mqm)
+    cp->mqm_ready_ptr = mqm->next;
+  GNUNET_CONTAINER_DLL_remove (cp->mqm_head,
+                               cp->mqm_tail,
+                               mqm);
+  GNUNET_free (mqm);
 }
 
 
 /**
- * Iterate all known peers.
+ * Send the message in @a env to @a cp, overriding queueing logic.
+ * This function should only be used to send error messages outside
+ * of flow and congestion control, similar to ICMP.  Note that
+ * the envelope may be silently discarded as well.
  *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
+ * @param cp peer to send the message to
+ * @param env envelope with the message to send
  */
 void
-GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
-                 void *cls)
+GCP_send_ooo (struct CadetPeer *cp,
+              struct GNUNET_MQ_Envelope *env)
 {
-    GCC_check_connections ();
-    GNUNET_CONTAINER_multipeermap_iterate (peers,
-                                           iter,
-                                           cls);
-    GCC_check_connections ();
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending message to %s out of management\n",
+       GCP_2s (cp));
+  if (NULL == cp->core_mq)
+  {
+    GNUNET_MQ_discard (env);
+    return;
+  }
+  GNUNET_MQ_notify_sent (env,
+                         &mqm_send_done,
+                         cp);
+  GNUNET_MQ_send (cp->core_mq,
+                  env);
 }
 
 
-/**
- * Get the static string for a peer ID.
- *
- * @param peer Peer.
- *
- * @return Static string for it's ID.
- */
-const char *
-GCP_2s (const struct CadetPeer *peer)
-{
-    if (NULL == peer)
-        return "(NULL)";
-    return GNUNET_i2s (GNUNET_PEER_resolve2 (peer->id));
-}
 
 
-/* end of gnunet-service-cadet_peer.c */
+/* end of gnunet-service-cadet-new_peer.c */
diff --git a/src/cadet/gnunet-service-cadet_peer.h b/src/cadet/gnunet-service-cadet_peer.h
index 1e206e10f..a2a6c6a92 100644
--- a/src/cadet/gnunet-service-cadet_peer.h
+++ b/src/cadet/gnunet-service-cadet_peer.h
@@ -1,6 +1,7 @@
+
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
+     Copyright (C) 2001-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -19,465 +20,375 @@
 */
 
 /**
- * @file cadet/gnunet-service-cadet_peer.h
- * @brief cadet service; dealing with remote peers
+ * @file cadet/gnunet-service-cadet-new_peer.h
+ * @brief Information we track per peer.
  * @author Bartlomiej Polot
- *
- * All functions in this file should use the prefix GMP (Gnunet Cadet Peer)
+ * @author Christian Grothoff
  */
-
 #ifndef GNUNET_SERVICE_CADET_PEER_H
 #define GNUNET_SERVICE_CADET_PEER_H
 
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "cadet_path.h"
-
-/**
- * Struct containing all information regarding a given peer
- */
-struct CadetPeer;
+#include "gnunet-service-cadet.h"
+#include "gnunet_hello_lib.h"
 
-/**
- * Handle to queued messages on a peer level.
- */
-struct CadetPeerQueue;
-
-#include "gnunet-service-cadet_connection.h"
-
-
-/**
- * Callback called when a queued message is sent.
- *
- * @param cls Closure.
- * @param c Connection this message was on.
- * @param fwd Was this a FWD going message?
- * @param sent Was it really sent? (Could have been canceled)
- * @param type Type of message sent.
- * @param payload_type Type of payload, if applicable.
- * @param pid Message ID, or 0 if not applicable (create, destroy, etc).
- * @param size Size of the message.
- * @param wait Time spent waiting for core (only the time for THIS message)
- */
-typedef void
-(*GCP_sent) (void *cls,
-             struct CadetConnection *c,
-             int fwd,
-             int sent,
-             uint16_t type,
-             uint16_t payload_type,
-             struct CadetEncryptedMessageIdentifier pid,
-             size_t size,
-             struct GNUNET_TIME_Relative wait);
 
 /**
- * Peer path iterator.
+ * Get the static string for a peer ID.
  *
- * @param cls Closure.
- * @param peer Peer this path is towards.
- * @param path Path itself
- * @return #GNUNET_YES if should keep iterating.
- *         #GNUNET_NO otherwise.
- */
-typedef int
-(*GCP_path_iterator) (void *cls,
-                      struct CadetPeer *peer,
-                      struct CadetPeerPath *path);
-
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Initialize peer subsystem.
+ * @param peer Peer.
  *
- * @param c Configuration.
- */
-void
-GCP_init (const struct GNUNET_CONFIGURATION_Handle *c);
-
-/**
- * Shut down the peer subsystem.
+ * @return Static string for it's ID.
  */
-void
-GCP_shutdown (void);
+const char *
+GCP_2s (const struct CadetPeer *peer);
 
 
 /**
- * Retrieve the CadetPeer stucture associated with the peer. Optionally create
- * one and insert it in the appropriate structures if the peer is not known yet.
+ * Retrieve the CadetPeer stucture associated with the
+ * peer. Optionally create one and insert it in the appropriate
+ * structures if the peer is not known yet.
  *
  * @param peer_id Full identity of the peer.
  * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO otherwise.
- *
+ *               #GNUNET_NO to return NULL if peer is unknown.
  * @return Existing or newly created peer structure.
  *         NULL if unknown and not requested @a create
  */
 struct CadetPeer *
-GCP_get (const struct GNUNET_PeerIdentity *peer_id, int create);
+GCP_get (const struct GNUNET_PeerIdentity *peer_id,
+         int create);
 
 
 /**
- * Retrieve the CadetPeer stucture associated with the peer. Optionally create
- * one and insert it in the appropriate structures if the peer is not known yet.
- *
- * @param peer Short identity of the peer.
- * @param create #GNUNET_YES if a new peer should be created if unknown.
- *               #GNUNET_NO otherwise.
+ * Calculate how desirable a path is for @a cp if
+ * @a cp is at offset @a off in the path.
  *
- * @return Existing or newly created peer structure.
- *         NULL if unknown and not requested @a create
+ * @param cp a peer reachable via a path
+ * @param off offset of @a cp in a path
+ * @return score how useful a path is to reach @a cp,
+ *         positive scores mean path is more desirable
  */
-struct CadetPeer *
-GCP_get_short (const GNUNET_PEER_Id peer, int create);
-
+double
+GCP_get_desirability_of_path (struct CadetPeer *cp,
+                              unsigned int off);
 
-/**
- * Try to establish a new connection to this peer (in its tunnel).
- * If the peer doesn't have any path to it yet, try to get one.
- * If the peer already has some path, send a CREATE CONNECTION towards it.
- *
- * @param peer Peer to connect to.
- */
-void
-GCP_connect (struct CadetPeer *peer);
 
 /**
- * @brief Send a message to another peer (using CORE).
+ * Obtain the peer identity for a `struct CadetPeer`.
  *
- * @param peer Peer towards which to queue the message.
- * @param message Message to send.
- * @param payload_type Type of the message's payload, for debug messages.
- *                     0 if the message is a retransmission (unknown payload).
- *                     UINT16_MAX if the message does not have payload.
- * @param payload_id ID of the payload (MID, ACK #, etc)
- * @param c Connection this message belongs to (can be NULL).
- * @param fwd Is this a message going root->dest? (FWD ACK are NOT FWD!)
- * @param cont Continuation to be called once CORE has sent the message.
- * @param cont_cls Closure for @c cont.
+ * @param cp our peer handle
+ * @return the peer identity
  */
-struct CadetPeerQueue *
-GCP_send (struct CadetPeer *peer,
-          const struct GNUNET_MessageHeader *message,
-          uint16_t payload_type,
-          struct CadetEncryptedMessageIdentifier payload_id,
-          struct CadetConnection *c,
-          int fwd,
-          GCP_sent cont,
-          void *cont_cls);
+const struct GNUNET_PeerIdentity *
+GCP_get_id (struct CadetPeer *cp);
 
-/**
- * Cancel sending a message. Message must have been sent with
- * #GCP_send before.  May not be called after the notify sent
- * callback has been called.
- *
- * It does NOT call the continuation given to #GCP_send.
- *
- * @param q Queue handle to cancel
- */
-void
-GCP_send_cancel (struct CadetPeerQueue *q);
 
 /**
- * Set tunnel.
+ * Iterate over all known peers.
  *
- * @param peer Peer.
- * @param t Tunnel.
+ * @param iter Iterator.
+ * @param cls Closure for @c iter.
  */
 void
-GCP_set_tunnel (struct CadetPeer *peer, struct CadetTunnel *t);
+GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
+                 void *cls);
 
 
 /**
- * Check whether there is a direct (core level)  connection to peer.
- *
- * @param peer Peer to check.
+ * Count the number of known paths toward the peer.
  *
- * @return #GNUNET_YES if there is a direct connection.
+ * @param cp Peer to get path info.
+ * @return Number of known paths.
  */
-int
-GCP_is_neighbor (const struct CadetPeer *peer);
+unsigned int
+GCP_count_paths (const struct CadetPeer *cp);
 
 
 /**
- * Create and initialize a new tunnel towards a peer, in case it has none.
- *
- * Does not generate any traffic, just creates the local data structures.
+ * Drop all paths owned by this peer, and do not
+ * allow new ones to be added: We are shutting down.
  *
- * @param peer Peer towards which to create the tunnel.
+ * @param cp peer to drop paths to
  */
 void
-GCP_add_tunnel (struct CadetPeer *peer);
+GCP_drop_owned_paths (struct CadetPeer *cp);
 
 
 /**
- * Add a connection to a neighboring peer.
- *
- * Store that the peer is the first hop of the connection in one
- * direction and that on peer disconnect the connection must be
- * notified and destroyed, for it will no longer be valid.
+ * Peer path iterator.
  *
- * @param peer Peer to add connection to.
- * @param c Connection to add.
- * @param pred #GNUNET_YES if we are predecessor, #GNUNET_NO if we are successor
+ * @param cls Closure.
+ * @param path Path itself
+ * @param off offset of the target peer in @a path
+ * @return #GNUNET_YES if should keep iterating.
+ *         #GNUNET_NO otherwise.
  */
-void
-GCP_add_connection (struct CadetPeer *peer,
-                    struct CadetConnection *c,
-                    int pred);
+typedef int
+(*GCP_PathIterator) (void *cls,
+                     struct CadetPeerPath *path,
+                     unsigned int off);
 
 
 /**
- * Add the path to the peer and update the path used to reach it in case this
- * is the shortest.
- *
- * @param peer Destination peer to add the path to.
- * @param path New path to add. Last peer must be the peer in arg 1.
- *             Path will be either used of freed if already known.
- * @param trusted Do we trust that this path is real?
+ * Iterate over the paths to a peer.
  *
- * @return path if path was taken, pointer to existing duplicate if exists
- *         NULL on error.
+ * @param cp Peer to get path info.
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
  */
-struct CadetPeerPath *
-GCP_add_path (struct CadetPeer *peer,
-              struct CadetPeerPath *p,
-              int trusted);
+unsigned int
+GCP_iterate_paths (struct CadetPeer *cp,
+                   GCP_PathIterator callback,
+                   void *callback_cls);
 
 
 /**
- * Add the path to the origin peer and update the path used to reach it in case
- * this is the shortest.
- * The path is given in peer_info -> destination, therefore we turn the path
- * upside down first.
+ * Iterate over the paths to @a peer where
+ * @a peer is at distance @a dist from us.
  *
- * @param peer Peer to add the path to, being the origin of the path.
- * @param path New path to add after being inversed.
- *             Path will be either used or freed.
- * @param trusted Do we trust that this path is real?
- *
- * @return path if path was taken, pointer to existing duplicate if exists
- *         NULL on error.
+ * @param cp Peer to get path info.
+ * @param dist desired distance of @a peer to us on the path
+ * @param callback Function to call for every path.
+ * @param callback_cls Closure for @a callback.
+ * @return Number of iterated paths.
  */
-struct CadetPeerPath *
-GCP_add_path_to_origin (struct CadetPeer *peer,
-                        struct CadetPeerPath *path,
-                        int trusted);
+unsigned int
+GCP_iterate_paths_at (struct CadetPeer *cp,
+                      unsigned int dist,
+                      GCP_PathIterator callback,
+                      void *callback_cls);
+
 
 /**
- * Adds a path to the info of all the peers in the path
+ * Remove an entry from the DLL of all of the paths that this peer is on.
  *
- * @param p Path to process.
- * @param confirmed Whether we know if the path works or not.
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
  */
 void
-GCP_add_path_to_all (const struct CadetPeerPath *p, int confirmed);
+GCP_path_entry_remove (struct CadetPeer *cp,
+                       struct CadetPeerPathEntry *entry,
+                       unsigned int off);
 
 
 /**
- * Remove any path to the peer that has the extact same peers as the one given.
+ * Add an entry to the DLL of all of the paths that this peer is on.
  *
- * @param peer Peer to remove the path from.
- * @param path Path to remove. Is always destroyed .
+ * @param cp peer to modify
+ * @param entry an entry on a path
+ * @param off offset of this peer on the path
  */
 void
-GCP_remove_path (struct CadetPeer *peer,
-                 struct CadetPeerPath *path);
+GCP_path_entry_add (struct CadetPeer *cp,
+                    struct CadetPeerPathEntry *entry,
+                    unsigned int off);
 
 
 /**
- * Check that we are aware of a connection from a neighboring peer.
+ * Get the tunnel towards a peer.
  *
- * @param peer Peer to the connection is with
- * @param c Connection that should be in the map with this peer.
+ * @param cp Peer to get from.
+ * @param create #GNUNET_YES to create a tunnel if we do not have one
+ * @return Tunnel towards peer.
  */
-void
-GCP_check_connection (const struct CadetPeer *peer,
-                      const struct CadetConnection *c);
+struct CadetTunnel *
+GCP_get_tunnel (struct CadetPeer *cp,
+                int create);
 
 
 /**
- * Remove a connection from a neighboring peer.
+ * The tunnel to the given peer no longer exists, remove it from our
+ * data structures, and possibly clean up the peer itself.
  *
- * @param peer Peer to remove connection from.
- * @param c Connection to remove.
+ * @param cp the peer affected
+ * @param t the dead tunnel
  */
 void
-GCP_remove_connection (struct CadetPeer *peer,
-                       const struct CadetConnection *c);
+GCP_drop_tunnel (struct CadetPeer *cp,
+                 struct CadetTunnel *t);
 
 
 /**
- * Start the DHT search for new paths towards the peer: we don't have
- * enough good connections.
+ * Try adding a @a path to this @a cp.  If the peer already
+ * has plenty of paths, return NULL.
  *
- * @param peer Destination peer.
+ * @param cp peer to which the @a path leads to
+ * @param path a path looking for an owner; may not be fully initialized yet!
+ * @param off offset of @a cp in @a path
+ * @param force for attaching the path
+ * @return NULL if this peer does not care to become a new owner,
+ *         otherwise the node in the peer's path heap for the @a path.
  */
-void
-GCP_start_search (struct CadetPeer *peer);
+struct GNUNET_CONTAINER_HeapNode *
+GCP_attach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 unsigned int off,
+                 int force);
 
 
 /**
- * Stop the DHT search for new paths towards the peer: we already have
- * enough good connections.
+ * This peer can no longer own @a path as the path
+ * has been extended and a peer further down the line
+ * is now the new owner.
  *
- * @param peer Destination peer.
+ * @param cp old owner of the @a path
+ * @param path path where the ownership is lost
+ * @param hn note in @a cp's path heap that must be deleted
  */
 void
-GCP_stop_search (struct CadetPeer *peer);
+GCP_detach_path (struct CadetPeer *cp,
+                 struct CadetPeerPath *path,
+                 struct GNUNET_CONTAINER_HeapNode *hn);
 
 
 /**
- * Get the Full ID of a peer.
- *
- * @param peer Peer to get from.
+ * Add a @a connection to this @a cp.
  *
- * @return Full ID of peer.
+ * @param cp peer via which the @a connection goes
+ * @param cc the connection to add
  */
-const struct GNUNET_PeerIdentity *
-GCP_get_id (const struct CadetPeer *peer);
+void
+GCP_add_connection (struct CadetPeer *cp,
+                    struct CadetConnection *cc);
 
 
 /**
- * Get the Short ID of a peer.
+ * Remove a @a connection that went via this @a cp.
  *
- * @param peer Peer to get from.
- *
- * @return Short ID of peer.
+ * @param cp peer via which the @a connection went
+ * @param cc the connection to remove
  */
-GNUNET_PEER_Id
-GCP_get_short_id (const struct CadetPeer *peer);
+void
+GCP_remove_connection (struct CadetPeer *cp,
+                       struct CadetConnection *cc);
 
 
 /**
- * Get the tunnel towards a peer.
- *
- * @param peer Peer to get from.
+ * We got a HELLO for a @a cp, remember it, and possibly
+ * trigger adequate actions (like trying to connect).
  *
- * @return Tunnel towards peer.
+ * @param cp the peer we got a HELLO for
+ * @param hello the HELLO to remember
  */
-struct CadetTunnel *
-GCP_get_tunnel (const struct CadetPeer *peer);
+void
+GCP_set_hello (struct CadetPeer *cp,
+               const struct GNUNET_HELLO_Message *hello);
 
 
 /**
- * Set the hello message.
- *
- * @param peer Peer whose message to set.
- * @param hello Hello message.
+ * Clean up all entries about all peers.
+ * Must only be called after all tunnels, CORE-connections and
+ * connections are down.
  */
 void
-GCP_set_hello (struct CadetPeer *peer,
-               const struct GNUNET_HELLO_Message *hello);
+GCP_destroy_all_peers (void);
 
 
 /**
- * Get the hello message.
- *
- * @param peer Peer whose message to get.
+ * Data structure used to track whom we have to notify about changes
+ * in our ability to transmit to a given peer.
  *
- * @return Hello message.
+ * All queue managers will be given equal chance for sending messages
+ * to @a cp.  This construct this guarantees fairness for access to @a
+ * cp among the different message queues.  Each connection or route
+ * will have its respective message queue managers for each direction.
  */
-struct GNUNET_HELLO_Message *
-GCP_get_hello (struct CadetPeer *peer);
+struct GCP_MessageQueueManager;
 
 
 /**
- * Try to connect to a peer on TRANSPORT level.
+ * Function to call with updated message queue object.
  *
- * @param peer Peer to whom to connect.
+ * @param cls closure
+ * @param available #GNUNET_YES if sending is now possible,
+ *                  #GNUNET_NO if sending is no longer possible
+ *                  #GNUNET_SYSERR if sending is no longer possible
+ *                                 and the last envelope was discarded
  */
-void
-GCP_try_connect (struct CadetPeer *peer);
+typedef void
+(*GCP_MessageQueueNotificationCallback)(void *cls,
+                                        int available);
+
 
 /**
- * Notify a peer that a link between two other peers is broken. If any path
- * used that link, eliminate it.
+ * Start message queue change notifications.  Will create a new slot
+ * to manage the message queue to the given @a cp.
  *
- * @param peer Peer affected by the change.
- * @param peer1 Peer whose link is broken.
- * @param peer2 Peer whose link is broken.
+ * @param cp peer to notify for
+ * @param cb function to call if mq becomes available or unavailable
+ * @param cb_cls closure for @a cb
+ * @return handle to cancel request
  */
-void
-GCP_notify_broken_link (struct CadetPeer *peer,
-                        const struct GNUNET_PeerIdentity *peer1,
-                        const struct GNUNET_PeerIdentity *peer2);
+struct GCP_MessageQueueManager *
+GCP_request_mq (struct CadetPeer *cp,
+                GCP_MessageQueueNotificationCallback cb,
+                void *cb_cls);
 
 
 /**
- * Count the number of known paths toward the peer.
- *
- * @param peer Peer to get path info.
+ * Test if @a cp has a core-level connection
  *
- * @return Number of known paths.
+ * @param cp peer to test
+ * @return #GNUNET_YES if @a cp has a core-level connection
  */
-unsigned int
-GCP_count_paths (const struct CadetPeer *peer);
+int
+GCP_has_core_connection (struct CadetPeer *cp);
+
 
 /**
- * Iterate over the paths to a peer.
- *
- * @param peer Peer to get path info.
- * @param callback Function to call for every path.
- * @param cls Closure for @a callback.
+ * Send the message in @a env via a @a mqm.  Must only be called at
+ * most once after the respective
+ * #GCP_MessageQueueNotificationCallback was called with `available`
+ * set to #GNUNET_YES, and not after the callback was called with
+ * `available` set to #GNUNET_NO or #GNUNET_SYSERR.
  *
- * @return Number of iterated paths.
+ * @param mqm message queue manager for the transmission
+ * @param env envelope with the message to send; must NOT
+ *            yet have a #GNUNET_MQ_notify_sent() callback attached to it
  */
-unsigned int
-GCP_iterate_paths (struct CadetPeer *peer,
-                   GCP_path_iterator callback,
-                   void *cls);
+void
+GCP_send (struct GCP_MessageQueueManager *mqm,
+          struct GNUNET_MQ_Envelope *env);
 
 
 /**
- * Iterate all known peers.
+ * Send the message in @a env to @a cp, overriding queueing logic.
+ * This function should only be used to send error messages outside
+ * of flow and congestion control, similar to ICMP.  Note that
+ * the envelope may be silently discarded as well.
  *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
+ * @param cp peer to send the message to
+ * @param env envelope with the message to send
  */
 void
-GCP_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter,
-                 void *cls);
+GCP_send_ooo (struct CadetPeer *cp,
+              struct GNUNET_MQ_Envelope *env);
 
 
 /**
- * Get the static string for a peer ID.
- *
- * @param peer Peer.
+ * Stops message queue change notifications and sends a last message.
+ * In practice, this is implemented by sending that @a last_env
+ * message immediately (if any), ignoring queue order.
  *
- * @return Static string for it's ID.
+ * @param mqm handle matching request to cancel
+ * @param last_env final message to transmit, or NULL
  */
-const char *
-GCP_2s (const struct CadetPeer *peer);
+void
+GCP_request_mq_cancel (struct GCP_MessageQueueManager *mqm,
+                       struct GNUNET_MQ_Envelope *last_env);
 
 
 /**
- * Log all kinds of info about a peer.
+ * Set the message queue to @a mq for peer @a cp and notify watchers.
  *
- * @param peer Peer.
+ * @param cp peer to modify
+ * @param mq message queue to set (can be NULL)
  */
 void
-GCP_debug (const struct CadetPeer *p,
-           enum GNUNET_ErrorType level);
-
+GCP_set_mq (struct CadetPeer *cp,
+            struct GNUNET_MQ_Handle *mq);
 
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
 
-/* ifndef GNUNET_CADET_SERVICE_PEER_H */
 #endif
-/* end of gnunet-cadet-service_peer.h */
diff --git a/src/cadet/gnunet-service-cadet_tunnel.c b/src/cadet/gnunet-service-cadet_tunnel.c
deleted file mode 100644
index a94e8f4ff..000000000
--- a/src/cadet/gnunet-service-cadet_tunnel.c
+++ /dev/null
@@ -1,3501 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/gnunet-service-cadet_tunnel.c
- * @brief logical links between CADET clients
- * @author Bartlomiej Polot
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_signatures.h"
-#include "gnunet_statistics_service.h"
-#include "cadet_protocol.h"
-#include "cadet_path.h"
-#include "gnunet-service-cadet_tunnel.h"
-#include "gnunet-service-cadet_connection.h"
-#include "gnunet-service-cadet_channel.h"
-#include "gnunet-service-cadet_peer.h"
-
-#define LOG(level, ...) GNUNET_log_from(level,"cadet-tun",__VA_ARGS__)
-#define LOG2(level, ...) GNUNET_log_from_nocheck(level,"cadet-tun",__VA_ARGS__)
-
-#define REKEY_WAIT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 5)
-
-#if !defined(GNUNET_CULL_LOGGING)
-  #define DUMP_KEYS_TO_STDERR GNUNET_YES
-#else
-  #define DUMP_KEYS_TO_STDERR GNUNET_NO
-#endif
-
-#define MIN_TUNNEL_BUFFER       8
-#define MAX_TUNNEL_BUFFER       64
-#define MAX_SKIPPED_KEYS        64
-#define MAX_KEY_GAP             256
-#define AX_HEADER_SIZE (sizeof (uint32_t) * 2\
-                        + sizeof (struct GNUNET_CRYPTO_EcdhePublicKey))
-
-/******************************************************************************/
-/********************************   STRUCTS  **********************************/
-/******************************************************************************/
-
-struct CadetTChannel
-{
-  struct CadetTChannel *next;
-  struct CadetTChannel *prev;
-  struct CadetChannel *ch;
-};
-
-
-/**
- * Entry in list of connections used by tunnel, with metadata.
- */
-struct CadetTConnection
-{
-  /**
-   * Next in DLL.
-   */
-  struct CadetTConnection *next;
-
-  /**
-   * Prev in DLL.
-   */
-  struct CadetTConnection *prev;
-
-  /**
-   * Connection handle.
-   */
-  struct CadetConnection *c;
-
-  /**
-   * Creation time, to keep oldest connection alive.
-   */
-  struct GNUNET_TIME_Absolute created;
-
-  /**
-   * Connection throughput, to keep fastest connection alive.
-   */
-  uint32_t throughput;
-};
-
-
-/**
- * Struct to old keys for skipped messages while advancing the Axolotl ratchet.
- */
-struct CadetTunnelSkippedKey
-{
-  /**
-   * DLL next.
-   */
-  struct CadetTunnelSkippedKey *next;
-
-  /**
-   * DLL prev.
-   */
-  struct CadetTunnelSkippedKey *prev;
-
-  /**
-   * When was this key stored (for timeout).
-   */
-  struct GNUNET_TIME_Absolute timestamp;
-
-  /**
-   * Header key.
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey HK;
-
-  /**
-   * Message key.
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey MK;
-
-  /**
-   * Key number for a given HK.
-   */
-  unsigned int Kn;
-};
-
-
-/**
- * Axolotl data, according to https://github.com/trevp/axolotl/wiki .
- */
-struct CadetTunnelAxolotl
-{
-  /**
-   * A (double linked) list of stored message keys and associated header keys
-   * for "skipped" messages, i.e. messages that have not been
-   * received despite the reception of more recent messages, (head).
-   */
-  struct CadetTunnelSkippedKey *skipped_head;
-
-  /**
-   * Skipped messages' keys DLL, tail.
-   */
-  struct CadetTunnelSkippedKey *skipped_tail;
-
-  /**
-   * Elements in @a skipped_head <-> @a skipped_tail.
-   */
-  unsigned int skipped;
-
-  /**
-   * 32-byte root key which gets updated by DH ratchet.
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey RK;
-
-  /**
-   * 32-byte header key (send).
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey HKs;
-
-  /**
-   * 32-byte header key (recv)
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey HKr;
-
-  /**
-   * 32-byte next header key (send).
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey NHKs;
-
-  /**
-   * 32-byte next header key (recv).
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey NHKr;
-
-  /**
-   * 32-byte chain keys (used for forward-secrecy updating, send).
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey CKs;
-
-  /**
-   * 32-byte chain keys (used for forward-secrecy updating, recv).
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey CKr;
-
-  /**
-   * ECDH for key exchange (A0 / B0).
-   */
-  struct GNUNET_CRYPTO_EcdhePrivateKey *kx_0;
-
-  /**
-   * ECDH Ratchet key (send).
-   */
-  struct GNUNET_CRYPTO_EcdhePrivateKey *DHRs;
-
-  /**
-   * ECDH Ratchet key (recv).
-   */
-  struct GNUNET_CRYPTO_EcdhePublicKey DHRr;
-
-  /**
-   * Message number (reset to 0 with each new ratchet, next message to send).
-   */
-  uint32_t Ns;
-
-  /**
-   * Message number (reset to 0 with each new ratchet, next message to recv).
-   */
-  uint32_t Nr;
-
-  /**
-   * Previous message numbers (# of msgs sent under prev ratchet)
-   */
-  uint32_t PNs;
-
-  /**
-   * True (#GNUNET_YES) if we have to send a new ratchet key in next msg.
-   */
-  int ratchet_flag;
-
-  /**
-   * Number of messages recieved since our last ratchet advance.
-   * - If this counter = 0, we cannot send a new ratchet key in next msg.
-   * - If this counter > 0, we can (but don't yet have to) send a new key.
-   */
-  unsigned int ratchet_allowed;
-
-  /**
-   * Number of messages recieved since our last ratchet advance.
-   * - If this counter = 0, we cannot send a new ratchet key in next msg.
-   * - If this counter > 0, we can (but don't yet have to) send a new key.
-   */
-  unsigned int ratchet_counter;
-
-  /**
-   * When does this ratchet expire and a new one is triggered.
-   */
-  struct GNUNET_TIME_Absolute ratchet_expiration;
-};
-
-
-/**
- * Struct containing all information regarding a tunnel to a peer.
- */
-struct CadetTunnel
-{
-  /**
-   * Endpoint of the tunnel.
-   */
-  struct CadetPeer *peer;
-
-  /**
-   * Axolotl info.
-   */
-  struct CadetTunnelAxolotl *ax;
-
-  /**
-   * State of the tunnel connectivity.
-   */
-  enum CadetTunnelCState cstate;
-
-  /**
-   * State of the tunnel encryption.
-   */
-  enum CadetTunnelEState estate;
-
-  /**
-   * Peer's ephemeral key, to recreate @c e_key and @c d_key when own ephemeral
-   * key changes.
-   */
-  struct GNUNET_CRYPTO_EcdhePublicKey peers_ephemeral_key;
-
-  /**
-   * Encryption ("our") key. It is only "confirmed" if kx_ctx is NULL.
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey e_key;
-
-  /**
-   * Decryption ("their") key. It is only "confirmed" if kx_ctx is NULL.
-   */
-  struct GNUNET_CRYPTO_SymmetricSessionKey d_key;
-
-  /**
-   * Task to start the rekey process.
-   */
-  struct GNUNET_SCHEDULER_Task *rekey_task;
-
-  /**
-   * Paths that are actively used to reach the destination peer.
-   */
-  struct CadetTConnection *connection_head;
-  struct CadetTConnection *connection_tail;
-
-  /**
-   * Next connection number.
-   */
-  uint32_t next_cid;
-
-  /**
-   * Channels inside this tunnel.
-   */
-  struct CadetTChannel *channel_head;
-  struct CadetTChannel *channel_tail;
-
-  /**
-   * Channel ID for the next created channel.
-   */
-  struct GNUNET_CADET_ChannelTunnelNumber next_ctn;
-
-  /**
-   * Destroy flag: if true, destroy on last message.
-   */
-  struct GNUNET_SCHEDULER_Task * destroy_task;
-
-  /**
-   * Queued messages, to transmit once tunnel gets connected.
-   */
-  struct CadetTunnelDelayed *tq_head;
-  struct CadetTunnelDelayed *tq_tail;
-
-  /**
-   * Task to trim connections if too many are present.
-   */
-  struct GNUNET_SCHEDULER_Task * trim_connections_task;
-
-  /**
-   * Ephemeral message in the queue (to avoid queueing more than one).
-   */
-  struct CadetConnectionQueue *ephm_h;
-
-  /**
-   * Pong message in the queue.
-   */
-  struct CadetConnectionQueue *pong_h;
-};
-
-
-/**
- * Struct used to save messages in a non-ready tunnel to send once connected.
- */
-struct CadetTunnelDelayed
-{
-  /**
-   * DLL
-   */
-  struct CadetTunnelDelayed *next;
-  struct CadetTunnelDelayed *prev;
-
-  /**
-   * Tunnel.
-   */
-  struct CadetTunnel *t;
-
-  /**
-   * Tunnel queue given to the channel to cancel request. Update on send_queued.
-   */
-  struct CadetTunnelQueue *tq;
-
-  /**
-   * Message to send.
-   */
-  /* struct GNUNET_MessageHeader *msg; */
-};
-
-
-/**
- * Handle for messages queued but not yet sent.
- */
-struct CadetTunnelQueue
-{
-  /**
-   * Connection queue handle, to cancel if necessary.
-   */
-  struct CadetConnectionQueue *cq;
-
-  /**
-   * Handle in case message hasn't been given to a connection yet.
-   */
-  struct CadetTunnelDelayed *tqd;
-
-  /**
-   * Continuation to call once sent.
-   */
-  GCT_sent cont;
-
-  /**
-   * Closure for @c cont.
-   */
-  void *cont_cls;
-};
-
-
-/******************************************************************************/
-/*******************************   GLOBALS  ***********************************/
-/******************************************************************************/
-
-/**
- * Global handle to the statistics service.
- */
-extern struct GNUNET_STATISTICS_Handle *stats;
-
-/**
- * Local peer own ID (memory efficient handle).
- */
-extern GNUNET_PEER_Id myid;
-
-/**
- * Local peer own ID (full value).
- */
-extern struct GNUNET_PeerIdentity my_full_id;
-
-
-/**
- * Don't try to recover tunnels if shutting down.
- */
-extern int shutting_down;
-
-
-/**
- * Set of all tunnels, in order to trigger a new exchange on rekey.
- * Indexed by peer's ID.
- */
-static struct GNUNET_CONTAINER_MultiPeerMap *tunnels;
-
-/**
- * Own Peer ID private key.
- */
-const static struct GNUNET_CRYPTO_EddsaPrivateKey *id_key;
-
-
-/********************************  AXOLOTL ************************************/
-
-/**
- * How many messages are needed to trigger a ratchet advance.
- */
-static unsigned long long ratchet_messages;
-
-/**
- * How long until we trigger a ratched advance.
- */
-static struct GNUNET_TIME_Relative ratchet_time;
-
-
-/******************************************************************************/
-/********************************   STATIC  ***********************************/
-/******************************************************************************/
-
-/**
- * Get string description for tunnel connectivity state.
- *
- * @param cs Tunnel state.
- *
- * @return String representation.
- */
-static const char *
-cstate2s (enum CadetTunnelCState cs)
-{
-  static char buf[32];
-
-  switch (cs)
-  {
-    case CADET_TUNNEL_NEW:
-      return "CADET_TUNNEL_NEW";
-    case CADET_TUNNEL_SEARCHING:
-      return "CADET_TUNNEL_SEARCHING";
-    case CADET_TUNNEL_WAITING:
-      return "CADET_TUNNEL_WAITING";
-    case CADET_TUNNEL_READY:
-      return "CADET_TUNNEL_READY";
-    case CADET_TUNNEL_SHUTDOWN:
-      return "CADET_TUNNEL_SHUTDOWN";
-    default:
-      SPRINTF (buf, "%u (UNKNOWN STATE)", cs);
-      return buf;
-  }
-  return "";
-}
-
-
-/**
- * Get string description for tunnel encryption state.
- *
- * @param es Tunnel state.
- *
- * @return String representation.
- */
-static const char *
-estate2s (enum CadetTunnelEState es)
-{
-  static char buf[32];
-
-  switch (es)
-  {
-    case CADET_TUNNEL_KEY_UNINITIALIZED:
-      return "CADET_TUNNEL_KEY_UNINITIALIZED";
-    case CADET_TUNNEL_KEY_AX_SENT:
-      return "CADET_TUNNEL_KEY_AX_SENT";
-    case CADET_TUNNEL_KEY_AX_AUTH_SENT:
-      return "CADET_TUNNEL_KEY_AX_AUTH_SENT";
-    case CADET_TUNNEL_KEY_OK:
-      return "CADET_TUNNEL_KEY_OK";
-    case CADET_TUNNEL_KEY_REKEY:
-      return "CADET_TUNNEL_KEY_REKEY";
-    default:
-      SPRINTF (buf, "%u (UNKNOWN STATE)", es);
-      return buf;
-  }
-  return "";
-}
-
-
-/**
- * @brief Check if tunnel is ready to send traffic.
- *
- * Tunnel must be connected and with encryption correctly set up.
- *
- * @param t Tunnel to check.
- *
- * @return #GNUNET_YES if ready, #GNUNET_NO otherwise
- */
-static int
-is_ready (struct CadetTunnel *t)
-{
-  int ready;
-  int conn_ok;
-  int enc_ok;
-
-  conn_ok = CADET_TUNNEL_READY == t->cstate;
-  enc_ok = CADET_TUNNEL_KEY_OK == t->estate
-           || CADET_TUNNEL_KEY_REKEY == t->estate
-           || CADET_TUNNEL_KEY_AX_AUTH_SENT == t->estate;
-  ready = conn_ok && enc_ok;
-  ready = ready || GCT_is_loopback (t);
-  return ready;
-}
-
-
-/**
- * Get the channel's buffer. ONLY FOR NON-LOOPBACK CHANNELS!!
- *
- * @param tch Tunnel's channel handle.
- *
- * @return Amount of messages the channel can still buffer towards the client.
- */
-static unsigned int
-get_channel_buffer (const struct CadetTChannel *tch)
-{
-  int fwd;
-
-  /* If channel is incoming, is terminal in the FWD direction and fwd is YES */
-  fwd = GCCH_is_terminal (tch->ch, GNUNET_YES);
-
-  return GCCH_get_buffer (tch->ch, fwd);
-}
-
-
-/**
- * Get the channel's allowance status.
- *
- * @param tch Tunnel's channel handle.
- *
- * @return #GNUNET_YES if we allowed the client to send data to us.
- */
-static int
-get_channel_allowed (const struct CadetTChannel *tch)
-{
-  int fwd;
-
-  /* If channel is outgoing, is origin in the FWD direction and fwd is YES */
-  fwd = GCCH_is_origin (tch->ch, GNUNET_YES);
-
-  return GCCH_get_allowed (tch->ch, fwd);
-}
-
-
-/**
- * Get the connection's buffer.
- *
- * @param tc Tunnel's connection handle.
- *
- * @return Amount of messages the connection can still buffer.
- */
-static unsigned int
-get_connection_buffer (const struct CadetTConnection *tc)
-{
-  int fwd;
-
-  /* If connection is outgoing, is origin in the FWD direction and fwd is YES */
-  fwd = GCC_is_origin (tc->c, GNUNET_YES);
-
-  return GCC_get_buffer (tc->c, fwd);
-}
-
-
-/**
- * Get the connection's allowance.
- *
- * @param tc Tunnel's connection handle.
- *
- * @return Amount of messages we have allowed the next peer to send us.
- */
-static unsigned int
-get_connection_allowed (const struct CadetTConnection *tc)
-{
-  int fwd;
-
-  /* If connection is outgoing, is origin in the FWD direction and fwd is YES */
-  fwd = GCC_is_origin (tc->c, GNUNET_YES);
-
-  return GCC_get_allowed (tc->c, fwd);
-}
-
-
-/**
- * Create a new Axolotl ephemeral (ratchet) key.
- *
- * @param t Tunnel.
- */
-static void
-new_ephemeral (struct CadetTunnel *t)
-{
-  GNUNET_free_non_null (t->ax->DHRs);
-  t->ax->DHRs = GNUNET_CRYPTO_ecdhe_key_create();
-  #if DUMP_KEYS_TO_STDERR
-  {
-    struct GNUNET_CRYPTO_EcdhePublicKey pub;
-    GNUNET_CRYPTO_ecdhe_key_get_public (t->ax->DHRs, &pub);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  new DHRs generated: pub  %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &pub));
-  }
-  #endif
-}
-
-
-/**
- * Calculate HMAC.
- *
- * @param plaintext Content to HMAC.
- * @param size Size of @c plaintext.
- * @param iv Initialization vector for the message.
- * @param key Key to use.
- * @param hmac[out] Destination to store the HMAC.
- */
-static void
-t_hmac (const void *plaintext, size_t size,
-        uint32_t iv, const struct GNUNET_CRYPTO_SymmetricSessionKey *key,
-        struct GNUNET_ShortHashCode *hmac)
-{
-  static const char ctx[] = "cadet authentication key";
-  struct GNUNET_CRYPTO_AuthKey auth_key;
-  struct GNUNET_HashCode hash;
-
-#if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_INFO, "  HMAC %u bytes with key %s\n", size,
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) key));
-#endif
-  GNUNET_CRYPTO_hmac_derive_key (&auth_key, key,
-                                 &iv, sizeof (iv),
-                                 key, sizeof (*key),
-                                 ctx, sizeof (ctx),
-                                 NULL);
-  /* Two step: CADET_Hash is only 256 bits, HashCode is 512. */
-  GNUNET_CRYPTO_hmac (&auth_key, plaintext, size, &hash);
-  GNUNET_memcpy (hmac, &hash, sizeof (*hmac));
-}
-
-
-/**
- * Perform a HMAC.
- *
- * @param key Key to use.
- * @param hash[out] Resulting HMAC.
- * @param source Source key material (data to HMAC).
- * @param len Length of @a source.
- */
-static void
-t_ax_hmac_hash (struct GNUNET_CRYPTO_SymmetricSessionKey *key,
-                struct GNUNET_HashCode *hash,
-                void *source, unsigned int len)
-{
-  static const char ctx[] = "axolotl HMAC-HASH";
-  struct GNUNET_CRYPTO_AuthKey auth_key;
-
-  GNUNET_CRYPTO_hmac_derive_key (&auth_key, key,
-                                 ctx, sizeof (ctx),
-                                 NULL);
-  GNUNET_CRYPTO_hmac (&auth_key, source, len, hash);
-}
-
-
-/**
- * Derive a key from a HMAC-HASH.
- *
- * @param key Key to use for the HMAC.
- * @param out Key to generate.
- * @param source Source key material (data to HMAC).
- * @param len Length of @a source.
- */
-static void
-t_hmac_derive_key (struct GNUNET_CRYPTO_SymmetricSessionKey *key,
-                   struct GNUNET_CRYPTO_SymmetricSessionKey *out,
-                   void *source, unsigned int len)
-{
-  static const char ctx[] = "axolotl derive key";
-  struct GNUNET_HashCode h;
-
-  t_ax_hmac_hash (key, &h, source, len);
-  GNUNET_CRYPTO_kdf (out, sizeof (*out), ctx, sizeof (ctx),
-                     &h, sizeof (h), NULL);
-}
-
-
-/**
- * Encrypt data with the axolotl tunnel key.
- *
- * @param t Tunnel whose key to use.
- * @param dst Destination for the encrypted data.
- * @param src Source of the plaintext. Can overlap with @c dst.
- * @param size Size of the plaintext.
- *
- * @return Size of the encrypted data.
- */
-static int
-t_ax_encrypt (struct CadetTunnel *t, void *dst, const void *src, size_t size)
-{
-  struct GNUNET_CRYPTO_SymmetricSessionKey MK;
-  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
-  size_t out_size;
-
-  CADET_TIMING_START;
-
-  ax = t->ax;
-  ax->ratchet_counter++;
-  if (GNUNET_YES == ax->ratchet_allowed
-      && (ratchet_messages <= ax->ratchet_counter
-          || 0 == GNUNET_TIME_absolute_get_remaining (ax->ratchet_expiration).rel_value_us))
-  {
-    ax->ratchet_flag = GNUNET_YES;
-  }
-
-  if (GNUNET_YES == ax->ratchet_flag)
-  {
-    /* Advance ratchet */
-    struct GNUNET_CRYPTO_SymmetricSessionKey keys[3];
-    struct GNUNET_HashCode dh;
-    struct GNUNET_HashCode hmac;
-    static const char ctx[] = "axolotl ratchet";
-
-    new_ephemeral (t);
-    ax->HKs = ax->NHKs;
-
-    /* RK, NHKs, CKs = KDF( HMAC-HASH(RK, DH(DHRs, DHRr)) ) */
-    GNUNET_CRYPTO_ecc_ecdh (ax->DHRs, &ax->DHRr, &dh);
-    t_ax_hmac_hash (&ax->RK, &hmac, &dh, sizeof (dh));
-    GNUNET_CRYPTO_kdf (keys, sizeof (keys), ctx, sizeof (ctx),
-                       &hmac, sizeof (hmac), NULL);
-    ax->RK = keys[0];
-    ax->NHKs = keys[1];
-    ax->CKs = keys[2];
-
-    ax->PNs = ax->Ns;
-    ax->Ns = 0;
-    ax->ratchet_flag = GNUNET_NO;
-    ax->ratchet_allowed = GNUNET_NO;
-    ax->ratchet_counter = 0;
-    ax->ratchet_expiration =
-      GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get(), ratchet_time);
-  }
-
-  t_hmac_derive_key (&ax->CKs, &MK, "0", 1);
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &MK, NULL, 0, NULL);
-
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  CKs: %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->CKs));
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_ENC with key %u: %s\n", ax->Ns,
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &MK));
-  #endif
-
-  out_size = GNUNET_CRYPTO_symmetric_encrypt (src, size, &MK, &iv, dst);
-  t_hmac_derive_key (&ax->CKs, &ax->CKs, "1", 1);
-
-  CADET_TIMING_END;
-
-  return out_size;
-}
-
-
-/**
- * Decrypt data with the axolotl tunnel key.
- *
- * @param t Tunnel whose key to use.
- * @param dst Destination for the decrypted data.
- * @param src Source of the ciphertext. Can overlap with @c dst.
- * @param size Size of the ciphertext.
- *
- * @return Size of the decrypted data.
- */
-static int
-t_ax_decrypt (struct CadetTunnel *t, void *dst, const void *src, size_t size)
-{
-  struct GNUNET_CRYPTO_SymmetricSessionKey MK;
-  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
-  size_t out_size;
-
-  CADET_TIMING_START;
-
-  ax = t->ax;
-
-  t_hmac_derive_key (&ax->CKr, &MK, "0", 1);
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &MK, NULL, 0, NULL);
-
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  CKr: %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->CKr));
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_DEC with key %u: %s\n", ax->Nr,
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &MK));
-  #endif
-
-  GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader));
-  out_size = GNUNET_CRYPTO_symmetric_decrypt (src, size, &MK, &iv, dst);
-  GNUNET_assert (out_size == size);
-
-  t_hmac_derive_key (&ax->CKr, &ax->CKr, "1", 1);
-
-  CADET_TIMING_END;
-
-  return out_size;
-}
-
-
-/**
- * Encrypt header with the axolotl header key.
- *
- * @param t Tunnel whose key to use.
- * @param msg Message whose header to encrypt.
- */
-static void
-t_h_encrypt (struct CadetTunnel *t, struct GNUNET_CADET_TunnelEncryptedMessage *msg)
-{
-  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
-  size_t out_size;
-
-  CADET_TIMING_START;
-  ax = t->ax;
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &ax->HKs, NULL, 0, NULL);
-
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_ENC_H with key %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->HKs));
-  #endif
-
-  out_size = GNUNET_CRYPTO_symmetric_encrypt (&msg->Ns, AX_HEADER_SIZE,
-                                              &ax->HKs, &iv, &msg->Ns);
-
-  GNUNET_assert (AX_HEADER_SIZE == out_size);
-  CADET_TIMING_END;
-}
-
-
-/**
- * Decrypt header with the current axolotl header key.
- *
- * @param t Tunnel whose current ax HK to use.
- * @param src Message whose header to decrypt.
- * @param dst Where to decrypt header to.
- */
-static void
-t_h_decrypt (struct CadetTunnel *t, const struct GNUNET_CADET_TunnelEncryptedMessage *src,
-             struct GNUNET_CADET_TunnelEncryptedMessage *dst)
-{
-  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct CadetTunnelAxolotl *ax;
-  size_t out_size;
-
-  CADET_TIMING_START;
-
-  ax = t->ax;
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &ax->HKr, NULL, 0, NULL);
-
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_DEC_H with key %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->HKr));
-  #endif
-
-  out_size = GNUNET_CRYPTO_symmetric_decrypt (&src->Ns, AX_HEADER_SIZE,
-                                              &ax->HKr, &iv, &dst->Ns);
-
-  GNUNET_assert (AX_HEADER_SIZE == out_size);
-
-  CADET_TIMING_END;
-}
-
-
-/**
- * Decrypt and verify data with the appropriate tunnel key and verify that the
- * data has not been altered since it was sent by the remote peer.
- *
- * @param t Tunnel whose key to use.
- * @param dst Destination for the plaintext.
- * @param src Source of the message. Can overlap with @c dst.
- * @param size Size of the message.
- *
- * @return Size of the decrypted data, -1 if an error was encountered.
- */
-static int
-try_old_ax_keys (struct CadetTunnel *t, void *dst,
-                 const struct GNUNET_CADET_TunnelEncryptedMessage *src, size_t size)
-{
-  struct CadetTunnelSkippedKey *key;
-  struct GNUNET_ShortHashCode *hmac;
-  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
-  struct GNUNET_CADET_TunnelEncryptedMessage plaintext_header;
-  struct GNUNET_CRYPTO_SymmetricSessionKey *valid_HK;
-  size_t esize;
-  size_t res;
-  size_t len;
-  unsigned int N;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Trying old keys\n");
-  hmac = &plaintext_header.hmac;
-  esize = size - sizeof (struct GNUNET_CADET_TunnelEncryptedMessage);
-
-  /* Find a correct Header Key */
-  for (key = t->ax->skipped_head; NULL != key; key = key->next)
-  {
-    #if DUMP_KEYS_TO_STDERR
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  Trying hmac with key %s\n",
-         GNUNET_i2s ((struct GNUNET_PeerIdentity *) &key->HK));
-    #endif
-    t_hmac (&src->Ns, AX_HEADER_SIZE + esize, 0, &key->HK, hmac);
-    if (0 == memcmp (hmac, &src->hmac, sizeof (*hmac)))
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  hmac correct\n");
-      valid_HK = &key->HK;
-      break;
-    }
-  }
-  if (NULL == key)
-    return -1;
-
-  /* Should've been checked in -cadet_connection.c handle_cadet_encrypted. */
-  GNUNET_assert (size > sizeof (struct GNUNET_CADET_TunnelEncryptedMessage));
-  len = size - sizeof (struct GNUNET_CADET_TunnelEncryptedMessage);
-  GNUNET_assert (len >= sizeof (struct GNUNET_MessageHeader));
-
-  /* Decrypt header */
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &key->HK, NULL, 0, NULL);
-  res = GNUNET_CRYPTO_symmetric_decrypt (&src->Ns, AX_HEADER_SIZE,
-                                         &key->HK, &iv, &plaintext_header.Ns);
-  GNUNET_assert (AX_HEADER_SIZE == res);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  Message %u, previous: %u\n",
-       ntohl (plaintext_header.Ns), ntohl (plaintext_header.PNs));
-
-  /* Find the correct Message Key */
-  N = ntohl (plaintext_header.Ns);
-  while (NULL != key && N != key->Kn)
-    key = key->next;
-  if (NULL == key || 0 != memcmp (&key->HK, valid_HK, sizeof (*valid_HK)))
-    return -1;
-
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_DEC_H with skipped key %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &key->HK));
-  LOG (GNUNET_ERROR_TYPE_INFO, "  AX_DEC with skipped key %u: %s\n",
-       key->Kn, GNUNET_i2s ((struct GNUNET_PeerIdentity *) &key->MK));
-  #endif
-
-  /* Decrypt payload */
-  GNUNET_CRYPTO_symmetric_derive_iv (&iv, &key->MK, NULL, 0, NULL);
-  res = GNUNET_CRYPTO_symmetric_decrypt (&src[1], len, &key->MK, &iv, dst);
-
-  /* Remove key */
-  GNUNET_CONTAINER_DLL_remove (t->ax->skipped_head, t->ax->skipped_tail, key);
-  t->ax->skipped--;
-  GNUNET_free (key); /* GNUNET_free overwrites memory with 0xbaadf00d */
-
-  return res;
-}
-
-
-/**
- * Delete a key from the list of skipped keys.
- *
- * @param t Tunnel to delete from.
- * @param HKr Header Key to use.
- */
-static void
-store_skipped_key (struct CadetTunnel *t,
-                   const struct GNUNET_CRYPTO_SymmetricSessionKey *HKr)
-{
-  struct CadetTunnelSkippedKey *key;
-
-  key = GNUNET_new (struct CadetTunnelSkippedKey);
-  key->timestamp = GNUNET_TIME_absolute_get ();
-  key->Kn = t->ax->Nr;
-  key->HK = t->ax->HKr;
-  t_hmac_derive_key (&t->ax->CKr, &key->MK, "0", 1);
-  #if DUMP_KEYS_TO_STDERR
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    storing MK for Nr %u: %s\n",
-       key->Kn, GNUNET_i2s ((struct GNUNET_PeerIdentity *) &key->MK));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    for CKr: %s\n",
-       GNUNET_i2s ((struct GNUNET_PeerIdentity *) &t->ax->CKr));
-  #endif
-  t_hmac_derive_key (&t->ax->CKr, &t->ax->CKr, "1", 1);
-  GNUNET_CONTAINER_DLL_insert (t->ax->skipped_head, t->ax->skipped_tail, key);
-  t->ax->Nr++;
-  t->ax->skipped++;
-}
-
-
-/**
- * Delete a key from the list of skipped keys.
- *
- * @param t Tunnel to delete from.
- * @param key Key to delete.
- */
-static void
-delete_skipped_key (struct CadetTunnel *t, struct CadetTunnelSkippedKey *key)
-{
-  GNUNET_CONTAINER_DLL_remove (t->ax->skipped_head, t->ax->skipped_tail, key);
-  GNUNET_free (key);
-  t->ax->skipped--;
-}
-
-
-/**
- * Stage skipped AX keys and calculate the message key.
- *
- * Stores each HK and MK for skipped messages.
- *
- * @param t Tunnel where to stage the keys.
- * @param HKr Header key.
- * @param Np Received meesage number.
- *
- * @return GNUNET_OK if keys were stored.
- *         GNUNET_SYSERR if an error ocurred (Np not expected).
- */
-static int
-store_ax_keys (struct CadetTunnel *t,
-               const struct GNUNET_CRYPTO_SymmetricSessionKey *HKr,
-               uint32_t Np)
-{
-  int gap;
-
-
-  gap = Np - t->ax->Nr;
-  LOG (GNUNET_ERROR_TYPE_INFO, "Storing keys [%u, %u)\n", t->ax->Nr, Np);
-  if (MAX_KEY_GAP < gap)
-  {
-    /* Avoid DoS (forcing peer to do 2*33 chain HMAC operations) */
-    /* TODO: start new key exchange on return */
-    GNUNET_break_op (0);
-    LOG (GNUNET_ERROR_TYPE_WARNING, "Got message %u, expected %u+\n",
-         Np, t->ax->Nr);
-    return GNUNET_SYSERR;
-  }
-  if (0 > gap)
-  {
-    /* Delayed message: don't store keys, flag to try old keys. */
-    return GNUNET_SYSERR;
-  }
-
-  while (t->ax->Nr < Np)
-    store_skipped_key (t, HKr);
-
-  while (t->ax->skipped > MAX_SKIPPED_KEYS)
-    delete_skipped_key (t, t->ax->skipped_tail);
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Decrypt and verify data with the appropriate tunnel key and verify that the
- * data has not been altered since it was sent by the remote peer.
- *
- * @param t Tunnel whose key to use.
- * @param dst Destination for the plaintext.
- * @param src Source of the message. Can overlap with @c dst.
- * @param size Size of the message.
- *
- * @return Size of the decrypted data, -1 if an error was encountered.
- */
-static int
-t_ax_decrypt_and_validate (struct CadetTunnel *t, void *dst,
-                           const struct GNUNET_CADET_TunnelEncryptedMessage *src,
-                           size_t size)
-{
-  struct CadetTunnelAxolotl *ax;
-  struct GNUNET_ShortHashCode msg_hmac;
-  struct GNUNET_HashCode hmac;
-  struct GNUNET_CADET_TunnelEncryptedMessage plaintext_header;
-  uint32_t Np;
-  uint32_t PNp;
-  size_t esize; /* Size of encryped payload */
-  size_t osize; /* Size of output (decrypted payload) */
-
-  esize = size - sizeof (struct GNUNET_CADET_TunnelEncryptedMessage);
-  ax = t->ax;
-  if (NULL == ax)
-    return -1;
-
-  /* Try current HK */
-  t_hmac (&src->Ns, AX_HEADER_SIZE + esize, 0, &ax->HKr, &msg_hmac);
-  if (0 != memcmp (&msg_hmac, &src->hmac, sizeof (msg_hmac)))
-  {
-    static const char ctx[] = "axolotl ratchet";
-    struct GNUNET_CRYPTO_SymmetricSessionKey keys[3]; /* RKp, NHKp, CKp */
-    struct GNUNET_CRYPTO_SymmetricSessionKey HK;
-    struct GNUNET_HashCode dh;
-    struct GNUNET_CRYPTO_EcdhePublicKey *DHRp;
-
-    /* Try Next HK */
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  trying next HK\n");
-    t_hmac (&src->Ns, AX_HEADER_SIZE + esize, 0, &ax->NHKr, &msg_hmac);
-    if (0 != memcmp (&msg_hmac, &src->hmac, sizeof (msg_hmac)))
-    {
-      /* Try the skipped keys, if that fails, we're out of luck. */
-      return try_old_ax_keys (t, dst, src, size);
-    }
-    LOG (GNUNET_ERROR_TYPE_INFO, "next HK worked\n");
-
-    HK = ax->HKr;
-    ax->HKr = ax->NHKr;
-    t_h_decrypt (t, src, &plaintext_header);
-    Np = ntohl (plaintext_header.Ns);
-    PNp = ntohl (plaintext_header.PNs);
-    DHRp = &plaintext_header.DHRs;
-    store_ax_keys (t, &HK, PNp);
-
-    /* RKp, NHKp, CKp = KDF (HMAC-HASH (RK, DH (DHRp, DHRs))) */
-    GNUNET_CRYPTO_ecc_ecdh (ax->DHRs, DHRp, &dh);
-    t_ax_hmac_hash (&ax->RK, &hmac, &dh, sizeof (dh));
-    GNUNET_CRYPTO_kdf (keys, sizeof (keys), ctx, sizeof (ctx),
-                       &hmac, sizeof (hmac), NULL);
-
-    /* Commit "purported" keys */
-    ax->RK = keys[0];
-    ax->NHKr = keys[1];
-    ax->CKr = keys[2];
-    ax->DHRr = *DHRp;
-    ax->Nr = 0;
-    ax->ratchet_allowed = GNUNET_YES;
-  }
-  else
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "current HK\n");
-    t_h_decrypt (t, src, &plaintext_header);
-    Np = ntohl (plaintext_header.Ns);
-    PNp = ntohl (plaintext_header.PNs);
-  }
-  LOG (GNUNET_ERROR_TYPE_INFO, "  got AX Nr %u\n", Np);
-  if (Np != ax->Nr)
-    if (GNUNET_OK != store_ax_keys (t, &ax->HKr, Np))
-      /* Try the skipped keys, if that fails, we're out of luck. */
-      return try_old_ax_keys (t, dst, src, size);
-
-  osize = t_ax_decrypt (t, dst, &src[1], esize);
-  ax->Nr = Np + 1;
-
-  if (osize != esize)
-  {
-    GNUNET_break_op (0);
-    return -1;
-  }
-
-  return osize;
-}
-
-
-/**
- * Pick a connection on which send the next data message.
- *
- * @param t Tunnel on which to send the message.
- *
- * @return The connection on which to send the next message.
- */
-static struct CadetConnection *
-tunnel_get_connection (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter;
-  struct CadetConnection *best;
-  unsigned int qn;
-  unsigned int lowest_q;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "tunnel_get_connection %s\n", GCT_2s (t));
-  best = NULL;
-  lowest_q = UINT_MAX;
-  for (iter = t->connection_head; NULL != iter; iter = iter->next)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  connection %s: %u\n",
-         GCC_2s (iter->c), GCC_get_state (iter->c));
-    if (CADET_CONNECTION_READY == GCC_get_state (iter->c))
-    {
-      qn = GCC_get_qn (iter->c, GCC_is_origin (iter->c, GNUNET_YES));
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "    q_n %u, \n", qn);
-      if (qn < lowest_q)
-      {
-        best = iter->c;
-        lowest_q = qn;
-      }
-    }
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " selected: connection %s\n", GCC_2s (best));
-  return best;
-}
-
-
-/**
- * Callback called when a queued message is sent.
- *
- * Calculates the average time and connection packet tracking.
- *
- * @param cls Closure (TunnelQueue handle).
- * @param c Connection this message was on.
- * @param q Connection queue handle (unused).
- * @param type Type of message sent.
- * @param fwd Was this a FWD going message?
- * @param size Size of the message.
- */
-static void
-tun_message_sent (void *cls,
-              struct CadetConnection *c,
-              struct CadetConnectionQueue *q,
-              uint16_t type, int fwd, size_t size)
-{
-  struct CadetTunnelQueue *qt = cls;
-  struct CadetTunnel *t;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "tun_message_sent\n");
-
-  GNUNET_assert (NULL != qt->cont);
-  t = NULL == c ? NULL : GCC_get_tunnel (c);
-  qt->cont (qt->cont_cls, t, qt, type, size);
-  GNUNET_free (qt);
-}
-
-
-static unsigned int
-count_queued_data (const struct CadetTunnel *t)
-{
-  struct CadetTunnelDelayed *iter;
-  unsigned int count;
-
-  for (count = 0, iter = t->tq_head; iter != NULL; iter = iter->next)
-    count++;
-
-  return count;
-}
-
-/**
- * Delete a queued message: either was sent or the channel was destroyed
- * before the tunnel's key exchange had a chance to finish.
- *
- * @param tqd Delayed queue handle.
- */
-static void
-unqueue_data (struct CadetTunnelDelayed *tqd)
-{
-  GNUNET_CONTAINER_DLL_remove (tqd->t->tq_head, tqd->t->tq_tail, tqd);
-  GNUNET_free (tqd);
-}
-
-
-/**
- * Cache a message to be sent once tunnel is online.
- *
- * @param t Tunnel to hold the message.
- * @param msg Message itself (copy will be made).
- */
-static struct CadetTunnelDelayed *
-queue_data (struct CadetTunnel *t, const struct GNUNET_MessageHeader *msg)
-{
-  struct CadetTunnelDelayed *tqd;
-  uint16_t size = ntohs (msg->size);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "queue data on Tunnel %s\n", GCT_2s (t));
-
-  GNUNET_assert (GNUNET_NO == is_ready (t));
-
-  tqd = GNUNET_malloc (sizeof (struct CadetTunnelDelayed) + size);
-
-  tqd->t = t;
-  GNUNET_memcpy (&tqd[1], msg, size);
-  GNUNET_CONTAINER_DLL_insert_tail (t->tq_head, t->tq_tail, tqd);
-  return tqd;
-}
-
-
-/**
- * Sends an already built message on a tunnel, encrypting it and
- * choosing the best connection.
- *
- * @param message Message to send. Function modifies it.
- * @param t Tunnel on which this message is transmitted.
- * @param c Connection to use (autoselect if NULL).
- * @param force Force the tunnel to take the message (buffer overfill).
- * @param cont Continuation to call once message is really sent.
- * @param cont_cls Closure for @c cont.
- * @param existing_q In case this a transmission of previously queued data,
- *                   this should be TunnelQueue given to the client.
- *                   Otherwise, NULL.
- * @return Handle to cancel message.
- *         NULL if @c cont is NULL or an error happens and message is dropped.
- */
-static struct CadetTunnelQueue *
-send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                       struct CadetTunnel *t,
-                       struct CadetConnection *c,
-                       int force,
-                       GCT_sent cont,
-                       void *cont_cls,
-                       struct CadetTunnelQueue *existing_q)
-{
-  struct GNUNET_MessageHeader *msg;
-  struct GNUNET_CADET_TunnelEncryptedMessage *ax_msg;
-  struct CadetTunnelQueue *tq;
-  size_t size = ntohs (message->size);
-  char cbuf[sizeof (struct GNUNET_CADET_TunnelEncryptedMessage) + size] GNUNET_ALIGN;
-  size_t esize;
-  uint16_t type;
-  int fwd;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GMT Send on Tunnel %s\n", GCT_2s (t));
-
-  if (GNUNET_NO == is_ready (t))
-  {
-    struct CadetTunnelDelayed *tqd;
-    /* A non null existing_q indicates sending of queued data.
-     * Should only happen after tunnel becomes ready.
-     */
-    GNUNET_assert (NULL == existing_q);
-    tqd = queue_data (t, message);
-    if (NULL == cont)
-      return NULL;
-    tq = GNUNET_new (struct CadetTunnelQueue);
-    tq->tqd = tqd;
-    tqd->tq = tq;
-    tq->cont = cont;
-    tq->cont_cls = cont_cls;
-    return tq;
-  }
-
-  GNUNET_assert (GNUNET_NO == GCT_is_loopback (t));
-
-  ax_msg = (struct GNUNET_CADET_TunnelEncryptedMessage *) cbuf;
-  msg = &ax_msg->header;
-  msg->size = htons (sizeof (struct GNUNET_CADET_TunnelEncryptedMessage) + size);
-  msg->type = htons (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_ENCRYPTED);
-  esize = t_ax_encrypt (t, &ax_msg[1], message, size);
-  ax_msg->Ns = htonl (t->ax->Ns++);
-  ax_msg->PNs = htonl (t->ax->PNs);
-  GNUNET_CRYPTO_ecdhe_key_get_public (t->ax->DHRs, &ax_msg->DHRs);
-  t_h_encrypt (t, ax_msg);
-  t_hmac (&ax_msg->Ns, AX_HEADER_SIZE + esize, 0, &t->ax->HKs, &ax_msg->hmac);
-  GNUNET_assert (esize == size);
-
-  if (NULL == c)
-    c = tunnel_get_connection (t);
-  if (NULL == c)
-  {
-    /* Why is tunnel 'ready'? Should have been queued! */
-    if (NULL != t->destroy_task)
-    {
-      GNUNET_break (0);
-      GCT_debug (t, GNUNET_ERROR_TYPE_WARNING);
-    }
-    return NULL; /* Drop... */
-  }
-  fwd = GCC_is_origin (c, GNUNET_YES);
-  ax_msg->cid = *GCC_get_id (c);
-  ax_msg->cemi = GCC_get_pid (c, fwd);
-
-  type = htons (message->type);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending message of type %s with CEMI %u and CID %s\n",
-       GC_m2s (type),
-       htonl (ax_msg->cemi.pid),
-       GNUNET_sh2s (&ax_msg->cid.connection_of_tunnel));
-
-  if (NULL == cont)
-  {
-    (void) GCC_send_prebuilt_message (msg,
-                                      type,
-                                      ax_msg->cemi,
-                                      c,
-                                      fwd,
-                                      force, NULL, NULL);
-    return NULL;
-  }
-  if (NULL == existing_q)
-  {
-    tq = GNUNET_new (struct CadetTunnelQueue); /* FIXME valgrind: leak*/
-  }
-  else
-  {
-    tq = existing_q;
-    tq->tqd = NULL;
-  }
-  tq->cont = cont;
-  tq->cont_cls = cont_cls;
-  tq->cq = GCC_send_prebuilt_message (msg,
-                                      type,
-                                      ax_msg->cemi,
-                                      c,
-                                      fwd,
-                                      force,
-                                      &tun_message_sent, tq);
-  GNUNET_assert (NULL != tq->cq);
-
-  return tq;
-}
-
-
-/**
- * Send all cached messages that we can, tunnel is online.
- *
- * @param t Tunnel that holds the messages. Cannot be loopback.
- */
-static void
-send_queued_data (struct CadetTunnel *t)
-{
-  struct CadetTunnelDelayed *tqd;
-  struct CadetTunnelDelayed *next;
-  unsigned int room;
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "Send queued data, tunnel %s\n", GCT_2s (t));
-
-  if (GCT_is_loopback (t))
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  if (GNUNET_NO == is_ready (t))
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING, "  not ready yet: %s/%s\n",
-         estate2s (t->estate), cstate2s (t->cstate));
-    return;
-  }
-
-  room = GCT_get_connections_buffer (t);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  buffer space: %u\n", room);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  tq head: %p\n", t->tq_head);
-  for (tqd = t->tq_head; NULL != tqd && room > 0; tqd = next)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " sending queued data\n");
-    next = tqd->next;
-    room--;
-    send_prebuilt_message ((struct GNUNET_MessageHeader *) &tqd[1],
-                           tqd->t, NULL, GNUNET_YES,
-                           NULL != tqd->tq ? tqd->tq->cont : NULL,
-                           NULL != tqd->tq ? tqd->tq->cont_cls : NULL,
-                           tqd->tq);
-    unqueue_data (tqd);
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GCT_send_queued_data end\n", GCP_2s (t->peer));
-}
-
-
-/**
- * @brief Resend the KX until we complete the handshake.
- *
- * @param cls Closure (tunnel).
- */
-static void
-kx_resend (void *cls)
-{
-  struct CadetTunnel *t = cls;
-
-  t->rekey_task = NULL;
-  if (CADET_TUNNEL_KEY_OK == t->estate)
-  {
-    /* Should have been canceled on estate change */
-    GNUNET_break (0);
-    return;
-  }
-
-  GCT_send_kx (t, CADET_TUNNEL_KEY_AX_SENT >= t->estate);
-}
-
-
-/**
- * Callback called when a queued message is sent.
- *
- * @param cls Closure.
- * @param c Connection this message was on.
- * @param type Type of message sent.
- * @param fwd Was this a FWD going message?
- * @param size Size of the message.
- */
-static void
-ephm_sent (void *cls,
-           struct CadetConnection *c,
-           struct CadetConnectionQueue *q,
-           uint16_t type, int fwd, size_t size)
-{
-  struct CadetTunnel *t = cls;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "ephemeral sent %s\n", GC_m2s (type));
-
-  t->ephm_h = NULL;
-
-  if (CADET_TUNNEL_KEY_OK == t->estate)
-    return;
-
-  if (NULL != t->rekey_task)
-  {
-    GNUNET_break (0);
-    GCT_debug (t, GNUNET_ERROR_TYPE_WARNING);
-    GNUNET_SCHEDULER_cancel (t->rekey_task);
-  }
-  t->rekey_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
-                                                &kx_resend, t);
-
-}
-
-
-/**
- * Called only on shutdown, destroy every tunnel.
- *
- * @param cls Closure (unused).
- * @param key Current public key.
- * @param value Value in the hash map (tunnel).
- *
- * @return #GNUNET_YES, so we should continue to iterate,
- */
-static int
-destroy_iterator (void *cls,
-                const struct GNUNET_PeerIdentity *key,
-                void *value)
-{
-  struct CadetTunnel *t = value;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "GCT_shutdown destroying tunnel at %p\n", t);
-  GCT_destroy (t);
-  return GNUNET_YES;
-}
-
-
-/**
- * Notify remote peer that we don't know a channel he is talking about,
- * probably CHANNEL_DESTROY was missed.
- *
- * @param t Tunnel on which to notify.
- * @param gid ID of the channel.
- */
-static void
-send_channel_destroy (struct CadetTunnel *t,
-                      struct GNUNET_CADET_ChannelTunnelNumber gid)
-{
-  struct GNUNET_CADET_ChannelManageMessage msg;
-
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY);
-  msg.header.size = htons (sizeof (msg));
-  msg.ctn = gid;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "WARNING destroying unknown channel %u on tunnel %s\n",
-       ntohl (gid.cn),
-       GCT_2s (t));
-  send_prebuilt_message (&msg.header, t, NULL, GNUNET_YES, NULL, NULL, NULL);
-}
-
-
-/**
- * Demultiplex data per channel and call appropriate channel handler.
- *
- * @param t Tunnel on which the data came.
- * @param msg Data message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-static void
-handle_data (struct CadetTunnel *t,
-             const struct GNUNET_CADET_ChannelAppDataMessage *msg,
-             int fwd)
-{
-  struct CadetChannel *ch;
-  char buf[128];
-  size_t size;
-  uint16_t type;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size <
-      sizeof (struct GNUNET_CADET_ChannelAppDataMessage) +
-      sizeof (struct GNUNET_MessageHeader))
-  {
-    GNUNET_break (0);
-    return;
-  }
-  type = ntohs (msg[1].header.type);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " payload of type %s\n", GC_m2s (type));
-  SPRINTF (buf, "# received payload of type %hu", type);
-  GNUNET_STATISTICS_update (stats, buf, 1, GNUNET_NO);
-
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL == ch)
-  {
-    GNUNET_STATISTICS_update (stats,
-                              "# data on unknown channel",
-                              1,
-                              GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "channel 0x%X unknown\n",
-         ntohl (msg->ctn.cn));
-    send_channel_destroy (t, msg->ctn);
-    return;
-  }
-
-  GCCH_handle_data (ch, msg, fwd);
-}
-
-
-/**
- * Demultiplex data ACKs per channel and update appropriate channel buffer info.
- *
- * @param t Tunnel on which the DATA ACK came.
- * @param msg DATA ACK message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-static void
-handle_data_ack (struct CadetTunnel *t,
-                 const struct GNUNET_CADET_ChannelDataAckMessage *msg,
-                 int fwd)
-{
-  struct CadetChannel *ch;
-  size_t size;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size != sizeof (struct GNUNET_CADET_ChannelDataAckMessage))
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL == ch)
-  {
-    GNUNET_STATISTICS_update (stats, "# data ack on unknown channel",
-                              1, GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "WARNING channel %u unknown\n",
-         ntohl (msg->ctn.cn));
-    return;
-  }
-
-  GCCH_handle_data_ack (ch, msg, fwd);
-}
-
-
-/**
- * Handle channel create.
- *
- * @param t Tunnel on which the message came.
- * @param msg ChannelCreate message.
- */
-static void
-handle_ch_create (struct CadetTunnel *t,
-                  const struct GNUNET_CADET_ChannelOpenMessage *msg)
-{
-  struct CadetChannel *ch;
-  size_t size;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size != sizeof (struct GNUNET_CADET_ChannelOpenMessage))
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL != ch && ! GCT_is_loopback (t))
-  {
-    /* Probably a retransmission, safe to ignore */
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "   already exists...\n");
-  }
-  ch = GCCH_handle_create (t, msg);
-  if (NULL != ch)
-    GCT_add_channel (t, ch);
-}
-
-
-
-/**
- * Handle channel NACK: check correctness and call channel handler for NACKs.
- *
- * @param t Tunnel on which the NACK came.
- * @param msg NACK message.
- */
-static void
-handle_ch_nack (struct CadetTunnel *t,
-                const struct GNUNET_CADET_ChannelManageMessage *msg)
-{
-  struct CadetChannel *ch;
-  size_t size;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size != sizeof (struct GNUNET_CADET_ChannelManageMessage))
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL == ch)
-  {
-    GNUNET_STATISTICS_update (stats, "# channel NACK on unknown channel",
-                              1, GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "WARNING channel %u unknown\n",
-         ntohl (msg->ctn.cn));
-    return;
-  }
-
-  GCCH_handle_nack (ch);
-}
-
-
-/**
- * Handle a CHANNEL ACK (SYNACK/ACK).
- *
- * @param t Tunnel on which the CHANNEL ACK came.
- * @param msg CHANNEL ACK message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-static void
-handle_ch_ack (struct CadetTunnel *t,
-               const struct GNUNET_CADET_ChannelManageMessage *msg,
-               int fwd)
-{
-  struct CadetChannel *ch;
-  size_t size;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size != sizeof (struct GNUNET_CADET_ChannelManageMessage))
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL == ch)
-  {
-    GNUNET_STATISTICS_update (stats,
-                              "# channel ack on unknown channel",
-                              1,
-                              GNUNET_NO);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "WARNING channel %u unknown\n",
-         ntohl (msg->ctn.cn));
-    return;
-  }
-
-  GCCH_handle_ack (ch, msg, fwd);
-}
-
-
-/**
- * Handle a channel destruction message.
- *
- * @param t Tunnel on which the message came.
- * @param msg Channel destroy message.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-static void
-handle_ch_destroy (struct CadetTunnel *t,
-                   const struct GNUNET_CADET_ChannelManageMessage *msg,
-                   int fwd)
-{
-  struct CadetChannel *ch;
-  size_t size;
-
-  /* Check size */
-  size = ntohs (msg->header.size);
-  if (size != sizeof (struct GNUNET_CADET_ChannelManageMessage))
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  /* Check channel */
-  ch = GCT_get_channel (t, msg->ctn);
-  if (NULL == ch)
-  {
-    /* Probably a retransmission, safe to ignore */
-    return;
-  }
-
-  GCCH_handle_destroy (ch, msg, fwd);
-}
-
-
-/**
- * Free Axolotl data.
- *
- * @param t Tunnel.
- */
-static void
-destroy_ax (struct CadetTunnel *t)
-{
-  if (NULL == t->ax)
-    return;
-
-  GNUNET_free_non_null (t->ax->DHRs);
-  GNUNET_free_non_null (t->ax->kx_0);
-  while (NULL != t->ax->skipped_head)
-    delete_skipped_key (t, t->ax->skipped_head);
-  GNUNET_assert (0 == t->ax->skipped);
-
-  GNUNET_free (t->ax);
-  t->ax = NULL;
-
-  if (NULL != t->rekey_task)
-  {
-    GNUNET_SCHEDULER_cancel (t->rekey_task);
-    t->rekey_task = NULL;
-  }
-  if (NULL != t->ephm_h)
-  {
-    GCC_cancel (t->ephm_h);
-    t->ephm_h = NULL;
-  }
-}
-
-
-/**
- * Demultiplex by message type and call appropriate handler for a message
- * towards a channel of a local tunnel.
- *
- * @param t Tunnel this message came on.
- * @param msgh Message header.
- * @param fwd Is this message fwd? This only is meaningful in loopback channels.
- *            #GNUNET_YES if message is FWD on the respective channel (loopback)
- *            #GNUNET_NO if message is BCK on the respective channel (loopback)
- *            #GNUNET_SYSERR if message on a one-ended channel (remote)
- */
-static void
-handle_decrypted (struct CadetTunnel *t,
-                  const struct GNUNET_MessageHeader *msgh,
-                  int fwd)
-{
-  uint16_t type;
-  char buf[256];
-
-  type = ntohs (msgh->type);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "<-- %s on %s\n", GC_m2s (type), GCT_2s (t));
-  SPRINTF (buf, "# received encrypted of type %hu (%s)", type, GC_m2s (type));
-  GNUNET_STATISTICS_update (stats, buf, 1, GNUNET_NO);
-
-  switch (type)
-  {
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE:
-      /* Do nothing, connection aleady got updated. */
-      GNUNET_STATISTICS_update (stats, "# keepalives received", 1, GNUNET_NO);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA:
-      /* Don't send hop ACK, wait for client to ACK */
-      handle_data (t, (struct GNUNET_CADET_ChannelAppDataMessage *) msgh, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_APP_DATA_ACK:
-      handle_data_ack (t, (struct GNUNET_CADET_ChannelDataAckMessage *) msgh, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
-      handle_ch_create (t, (struct GNUNET_CADET_ChannelOpenMessage *) msgh);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED:
-      handle_ch_nack (t, (struct GNUNET_CADET_ChannelManageMessage *) msgh);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_ACK:
-      handle_ch_ack (t, (struct GNUNET_CADET_ChannelManageMessage *) msgh, fwd);
-      break;
-
-    case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
-      handle_ch_destroy (t, (struct GNUNET_CADET_ChannelManageMessage *) msgh, fwd);
-      break;
-
-    default:
-      GNUNET_break_op (0);
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           "end-to-end message not known (%u)\n",
-           ntohs (msgh->type));
-      GCT_debug (t, GNUNET_ERROR_TYPE_WARNING);
-  }
-}
-
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Decrypt and process an encrypted message.
- *
- * Calls the appropriate handler for a message in a channel of a local tunnel.
- *
- * @param t Tunnel this message came on.
- * @param msg Message header.
- */
-void
-GCT_handle_encrypted (struct CadetTunnel *t,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg)
-{
-  uint16_t size = ntohs (msg->header.size);
-  char cbuf [size];
-  int decrypted_size;
-  const struct GNUNET_MessageHeader *msgh;
-  unsigned int off;
-
-  GNUNET_STATISTICS_update (stats, "# received encrypted", 1, GNUNET_NO);
-
-  decrypted_size = t_ax_decrypt_and_validate (t, cbuf, msg, size);
-
-  if (-1 == decrypted_size)
-  {
-    GNUNET_STATISTICS_update (stats, "# unable to decrypt", 1, GNUNET_NO);
-    if (CADET_TUNNEL_KEY_AX_AUTH_SENT <= t->estate)
-    {
-      GNUNET_break_op (0);
-      LOG (GNUNET_ERROR_TYPE_WARNING, "Wrong crypto, tunnel %s\n", GCT_2s (t));
-      GCT_debug (t, GNUNET_ERROR_TYPE_WARNING);
-    }
-    return;
-  }
-  GCT_change_estate (t, CADET_TUNNEL_KEY_OK);
-
-  /* FIXME: this is bad, as the structs returned from
-     this loop may be unaligned, see util's MST for
-     how to do this right. */
-  off = 0;
-  while (off + sizeof (struct GNUNET_MessageHeader) <= decrypted_size)
-  {
-    uint16_t msize;
-
-    msgh = (const struct GNUNET_MessageHeader *) &cbuf[off];
-    msize = ntohs (msgh->size);
-    if (msize < sizeof (struct GNUNET_MessageHeader))
-    {
-      GNUNET_break_op (0);
-      return;
-    }
-    if (off + msize < decrypted_size)
-    {
-      GNUNET_break_op (0);
-      return;
-    }
-    handle_decrypted (t, msgh, GNUNET_SYSERR);
-    off += msize;
-  }
-}
-
-
-/**
- * Handle a Key eXchange message.
- *
- * @param t Tunnel on which the message came.
- * @param msg KX message itself.
- */
-void
-GCT_handle_kx (struct CadetTunnel *t,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg)
-{
-  struct CadetTunnelAxolotl *ax;
-  struct GNUNET_HashCode key_material[3];
-  struct GNUNET_CRYPTO_SymmetricSessionKey keys[5];
-  const char salt[] = "CADET Axolotl salt";
-  const struct GNUNET_PeerIdentity *pid;
-  int am_I_alice;
-
-  CADET_TIMING_START;
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "<== {        KX} on %s\n", GCT_2s (t));
-
-  if (NULL == t->ax)
-  {
-    /* Something is wrong if ax is NULL. Whose fault it is? */
-    return;
-  }
-  ax = t->ax;
-
-  pid = GCT_get_destination (t);
-  if (0 > GNUNET_CRYPTO_cmp_peer_identity (&my_full_id, pid))
-    am_I_alice = GNUNET_YES;
-  else if (0 < GNUNET_CRYPTO_cmp_peer_identity (&my_full_id, pid))
-    am_I_alice = GNUNET_NO;
-  else
-  {
-    GNUNET_break_op (0);
-    return;
-  }
-
-  if (0 != (GNUNET_CADET_KX_FLAG_FORCE_REPLY & ntohl (msg->flags)))
-  {
-    if (NULL != t->rekey_task)
-    {
-      GNUNET_SCHEDULER_cancel (t->rekey_task);
-      t->rekey_task = NULL;
-    }
-    GCT_send_kx (t, GNUNET_NO);
-  }
-
-  if (0 == memcmp (&ax->DHRr, &msg->ratchet_key, sizeof(msg->ratchet_key)))
-  {
-    LOG (GNUNET_ERROR_TYPE_INFO, " known ratchet key, exit\n");
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_INFO, " is Alice? %s\n", am_I_alice ? "YES" : "NO");
-
-  ax->DHRr = msg->ratchet_key;
-
-  /* ECDH A B0 */
-  if (GNUNET_YES == am_I_alice)
-  {
-    GNUNET_CRYPTO_eddsa_ecdh (id_key,              /* A */
-                              &msg->ephemeral_key, /* B0 */
-                              &key_material[0]);
-  }
-  else
-  {
-    GNUNET_CRYPTO_ecdh_eddsa (ax->kx_0,            /* B0 */
-                              &pid->public_key,    /* A */
-                              &key_material[0]);
-  }
-
-  /* ECDH A0 B */
-  if (GNUNET_YES == am_I_alice)
-  {
-    GNUNET_CRYPTO_ecdh_eddsa (ax->kx_0,            /* A0 */
-                              &pid->public_key,    /* B */
-                              &key_material[1]);
-  }
-  else
-  {
-    GNUNET_CRYPTO_eddsa_ecdh (id_key,              /* A */
-                              &msg->ephemeral_key, /* B0 */
-                              &key_material[1]);
-
-
-  }
-
-  /* ECDH A0 B0 */
-  /* (This is the triple-DH, we could probably safely skip this,
-     as A0/B0 are already in the key material.) */
-  GNUNET_CRYPTO_ecc_ecdh (ax->kx_0,             /* A0 or B0 */
-                          &msg->ephemeral_key,  /* B0 or A0 */
-                          &key_material[2]);
-
-  #if DUMP_KEYS_TO_STDERR
-  {
-    unsigned int i;
-    for (i = 0; i < 3; i++)
-      LOG (GNUNET_ERROR_TYPE_INFO, "km[%u]: %s\n",
-           i, GNUNET_h2s (&key_material[i]));
-  }
-  #endif
-
-  /* KDF */
-  GNUNET_CRYPTO_kdf (keys, sizeof (keys),
-                     salt, sizeof (salt),
-                     &key_material, sizeof (key_material), NULL);
-
-  if (0 == memcmp (&ax->RK, &keys[0], sizeof(ax->RK)))
-  {
-    LOG (GNUNET_ERROR_TYPE_INFO, " known handshake key, exit\n");
-    return;
-  }
-  ax->RK = keys[0];
-  if (GNUNET_YES == am_I_alice)
-  {
-    ax->HKr = keys[1];
-    ax->NHKs = keys[2];
-    ax->NHKr = keys[3];
-    ax->CKr = keys[4];
-    ax->ratchet_flag = GNUNET_YES;
-  }
-  else
-  {
-    ax->HKs = keys[1];
-    ax->NHKr = keys[2];
-    ax->NHKs = keys[3];
-    ax->CKs = keys[4];
-    ax->ratchet_flag = GNUNET_NO;
-    ax->ratchet_allowed = GNUNET_NO;
-    ax->ratchet_counter = 0;
-    ax->ratchet_expiration =
-      GNUNET_TIME_absolute_add (GNUNET_TIME_absolute_get(), ratchet_time);
-  }
-  ax->PNs = 0;
-  ax->Nr = 0;
-  ax->Ns = 0;
-
-  GCT_change_estate (t, CADET_TUNNEL_KEY_AX_AUTH_SENT);
-  send_queued_data (t);
-
-  CADET_TIMING_END;
-}
-
-/**
- * Initialize the tunnel subsystem.
- *
- * @param c Configuration handle.
- * @param key ECC private key, to derive all other keys and do crypto.
- */
-void
-GCT_init (const struct GNUNET_CONFIGURATION_Handle *c,
-          const struct GNUNET_CRYPTO_EddsaPrivateKey *key)
-{
-  unsigned int expected_overhead;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "init\n");
-
-  expected_overhead = 0;
-  expected_overhead += sizeof (struct GNUNET_CADET_TunnelEncryptedMessage);
-  expected_overhead += sizeof (struct GNUNET_CADET_ChannelAppDataMessage);
-  expected_overhead += sizeof (struct GNUNET_CADET_ConnectionEncryptedAckMessage);
-  GNUNET_assert (GNUNET_CONSTANTS_CADET_P2P_OVERHEAD == expected_overhead);
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (c,
-                                             "CADET",
-                                             "RATCHET_MESSAGES",
-                                             &ratchet_messages))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET",
-                               "RATCHET_MESSAGES",
-                               "USING DEFAULT");
-    ratchet_messages = 64;
-  }
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_time (c,
-                                           "CADET",
-                                           "RATCHET_TIME",
-                                           &ratchet_time))
-  {
-    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
-                               "CADET", "RATCHET_TIME", "USING DEFAULT");
-    ratchet_time = GNUNET_TIME_UNIT_HOURS;
-  }
-
-
-  id_key = key;
-  tunnels = GNUNET_CONTAINER_multipeermap_create (128, GNUNET_YES);
-}
-
-
-/**
- * Shut down the tunnel subsystem.
- */
-void
-GCT_shutdown (void)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Shutting down tunnels\n");
-  GNUNET_CONTAINER_multipeermap_iterate (tunnels, &destroy_iterator, NULL);
-  GNUNET_CONTAINER_multipeermap_destroy (tunnels);
-}
-
-
-/**
- * Create a tunnel.
- *
- * @param destination Peer this tunnel is towards.
- */
-struct CadetTunnel *
-GCT_new (struct CadetPeer *destination)
-{
-  struct CadetTunnel *t;
-
-  t = GNUNET_new (struct CadetTunnel);
-  t->next_ctn.cn = 0;
-  t->peer = destination;
-
-  if (GNUNET_OK !=
-      GNUNET_CONTAINER_multipeermap_put (tunnels, GCP_get_id (destination), t,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST))
-  {
-    GNUNET_break (0);
-    GNUNET_free (t);
-    return NULL;
-  }
-  t->ax = GNUNET_new (struct CadetTunnelAxolotl);
-  new_ephemeral (t);
-  t->ax->kx_0 = GNUNET_CRYPTO_ecdhe_key_create ();
-  return t;
-}
-
-
-/**
- * Change the tunnel's connection state.
- *
- * @param t Tunnel whose connection state to change.
- * @param cstate New connection state.
- */
-void
-GCT_change_cstate (struct CadetTunnel* t, enum CadetTunnelCState cstate)
-{
-  if (NULL == t)
-    return;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel %s cstate %s => %s\n",
-       GCP_2s (t->peer), cstate2s (t->cstate), cstate2s (cstate));
-  if (myid != GCP_get_short_id (t->peer) &&
-      CADET_TUNNEL_READY != t->cstate &&
-      CADET_TUNNEL_READY == cstate)
-  {
-    t->cstate = cstate;
-    if (CADET_TUNNEL_KEY_OK == t->estate)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  cstate triggered send queued data\n");
-      send_queued_data (t);
-    }
-    else if (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "  cstate triggered KX\n");
-      GCT_send_kx (t, GNUNET_NO);
-    }
-    else
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "estate %s\n", estate2s (t->estate));
-    }
-  }
-  t->cstate = cstate;
-
-  if (CADET_TUNNEL_READY == cstate
-      && CONNECTIONS_PER_TUNNEL <= GCT_count_connections (t))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  cstate triggered stop dht\n");
-    GCP_stop_search (t->peer);
-  }
-}
-
-
-/**
- * Change the tunnel encryption state.
- *
- * If the encryption state changes to OK, stop the rekey task.
- *
- * @param t Tunnel whose encryption state to change, or NULL.
- * @param state New encryption state.
- */
-void
-GCT_change_estate (struct CadetTunnel* t, enum CadetTunnelEState state)
-{
-  enum CadetTunnelEState old;
-
-  if (NULL == t)
-    return;
-
-  old = t->estate;
-  t->estate = state;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel %s estate was %s\n",
-       GCP_2s (t->peer), estate2s (old));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel %s estate is now %s\n",
-       GCP_2s (t->peer), estate2s (t->estate));
-
-  if (CADET_TUNNEL_KEY_OK != old && CADET_TUNNEL_KEY_OK == t->estate)
-  {
-    if (NULL != t->rekey_task)
-    {
-      GNUNET_SCHEDULER_cancel (t->rekey_task);
-      t->rekey_task = NULL;
-    }
-    /* Send queued data if tunnel is not loopback */
-    if (myid != GCP_get_short_id (t->peer))
-      send_queued_data (t);
-  }
-}
-
-
-/**
- * @brief Check if tunnel has too many connections, and remove one if necessary.
- *
- * Currently this means the newest connection, unless it is a direct one.
- * Implemented as a task to avoid freeing a connection that is in the middle
- * of being created/processed.
- *
- * @param cls Closure (Tunnel to check).
- */
-static void
-trim_connections (void *cls)
-{
-  struct CadetTunnel *t = cls;
-
-  t->trim_connections_task = NULL;
-  if (GCT_count_connections (t) > 2 * CONNECTIONS_PER_TUNNEL)
-  {
-    struct CadetTConnection *iter;
-    struct CadetTConnection *c;
-
-    for (c = iter = t->connection_head; NULL != iter; iter = iter->next)
-    {
-      if ((iter->created.abs_value_us > c->created.abs_value_us)
-          && GNUNET_NO == GCC_is_direct (iter->c))
-      {
-        c = iter;
-      }
-    }
-    if (NULL != c)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "Too many connections on tunnel %s\n",
-           GCT_2s (t));
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "Destroying connection %s\n",
-           GCC_2s (c->c));
-      GCC_destroy (c->c);
-    }
-    else
-    {
-      GNUNET_break (0);
-    }
-  }
-}
-
-
-/**
- * Add a connection to a tunnel.
- *
- * @param t Tunnel.
- * @param c Connection.
- */
-void
-GCT_add_connection (struct CadetTunnel *t, struct CadetConnection *c)
-{
-  struct CadetTConnection *aux;
-
-  GNUNET_assert (NULL != c);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "add connection %s\n", GCC_2s (c));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " to tunnel %s\n", GCT_2s (t));
-  for (aux = t->connection_head; aux != NULL; aux = aux->next)
-    if (aux->c == c)
-      return;
-
-  aux = GNUNET_new (struct CadetTConnection);
-  aux->c = c;
-  aux->created = GNUNET_TIME_absolute_get ();
-
-  GNUNET_CONTAINER_DLL_insert (t->connection_head, t->connection_tail, aux);
-
-  if (CADET_TUNNEL_SEARCHING == t->cstate)
-    GCT_change_cstate (t, CADET_TUNNEL_WAITING);
-
-  if (NULL != t->trim_connections_task)
-    t->trim_connections_task = GNUNET_SCHEDULER_add_now (&trim_connections, t);
-}
-
-
-/**
- * Remove a connection from a tunnel.
- *
- * @param t Tunnel.
- * @param c Connection.
- */
-void
-GCT_remove_connection (struct CadetTunnel *t,
-                       struct CadetConnection *c)
-{
-  struct CadetTConnection *aux;
-  struct CadetTConnection *next;
-  unsigned int conns;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Removing connection %s from tunnel %s\n",
-       GCC_2s (c), GCT_2s (t));
-  for (aux = t->connection_head; aux != NULL; aux = next)
-  {
-    next = aux->next;
-    if (aux->c == c)
-    {
-      GNUNET_CONTAINER_DLL_remove (t->connection_head, t->connection_tail, aux);
-      GNUNET_free (aux);
-    }
-  }
-
-  conns = GCT_count_connections (t);
-  if (0 == conns
-      && NULL == t->destroy_task
-      && CADET_TUNNEL_SHUTDOWN != t->cstate
-      && GNUNET_NO == shutting_down)
-  {
-    if (0 == GCT_count_any_connections (t))
-      GCT_change_cstate (t, CADET_TUNNEL_SEARCHING);
-    else
-      GCT_change_cstate (t, CADET_TUNNEL_WAITING);
-  }
-
-  /* Start new connections if needed */
-  if (CONNECTIONS_PER_TUNNEL > conns
-      && CADET_TUNNEL_SHUTDOWN != t->cstate
-      && GNUNET_NO == shutting_down)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  too few connections, getting new ones\n");
-    GCP_connect (t->peer); /* Will change cstate to WAITING when possible */
-    return;
-  }
-
-  /* If not marked as ready, no change is needed */
-  if (CADET_TUNNEL_READY != t->cstate)
-    return;
-
-  /* Check if any connection is ready to maintain cstate */
-  for (aux = t->connection_head; aux != NULL; aux = aux->next)
-    if (CADET_CONNECTION_READY == GCC_get_state (aux->c))
-      return;
-}
-
-
-/**
- * Add a channel to a tunnel.
- *
- * @param t Tunnel.
- * @param ch Channel.
- */
-void
-GCT_add_channel (struct CadetTunnel *t,
-                 struct CadetChannel *ch)
-{
-  struct CadetTChannel *aux;
-
-  GNUNET_assert (NULL != ch);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Adding channel %p to tunnel %p\n", ch, t);
-
-  for (aux = t->channel_head; aux != NULL; aux = aux->next)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  already there %p\n", aux->ch);
-    if (aux->ch == ch)
-      return;
-  }
-
-  aux = GNUNET_new (struct CadetTChannel);
-  aux->ch = ch;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       " adding %p to %p\n", aux, t->channel_head);
-  GNUNET_CONTAINER_DLL_insert_tail (t->channel_head,
-                                    t->channel_tail,
-                                    aux);
-
-  if (NULL != t->destroy_task)
-  {
-    GNUNET_SCHEDULER_cancel (t->destroy_task);
-    t->destroy_task = NULL;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " undo destroy!\n");
-  }
-}
-
-
-/**
- * Remove a channel from a tunnel.
- *
- * @param t Tunnel.
- * @param ch Channel.
- */
-void
-GCT_remove_channel (struct CadetTunnel *t, struct CadetChannel *ch)
-{
-  struct CadetTChannel *aux;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Removing channel %p from tunnel %p\n", ch, t);
-  for (aux = t->channel_head; aux != NULL; aux = aux->next)
-  {
-    if (aux->ch == ch)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG, " found! %s\n", GCCH_2s (ch));
-      GNUNET_CONTAINER_DLL_remove (t->channel_head,
-                                   t->channel_tail,
-                                   aux);
-      GNUNET_free (aux);
-      return;
-    }
-  }
-}
-
-
-/**
- * Search for a channel by global ID.
- *
- * @param t Tunnel containing the channel.
- * @param ctn Public channel number.
- *
- * @return channel handler, NULL if doesn't exist
- */
-struct CadetChannel *
-GCT_get_channel (struct CadetTunnel *t,
-                 struct GNUNET_CADET_ChannelTunnelNumber ctn)
-{
-  struct CadetTChannel *iter;
-
-  if (NULL == t)
-    return NULL;
-
-  for (iter = t->channel_head; NULL != iter; iter = iter->next)
-  {
-    if (GCCH_get_id (iter->ch).cn == ctn.cn)
-      break;
-  }
-
-  return NULL == iter ? NULL : iter->ch;
-}
-
-
-/**
- * @brief Destroy a tunnel and free all resources.
- *
- * Should only be called a while after the tunnel has been marked as destroyed,
- * in case there is a new channel added to the same peer shortly after marking
- * the tunnel. This way we avoid a new public key handshake.
- *
- * @param cls Closure (tunnel to destroy).
- */
-static void
-delayed_destroy (void *cls)
-{
-  struct CadetTunnel *t = cls;
-  struct CadetTConnection *iter;
-
-  t->destroy_task = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "delayed destroying tunnel %p\n",
-       t);
-  t->cstate = CADET_TUNNEL_SHUTDOWN;
-  for (iter = t->connection_head; NULL != iter; iter = iter->next)
-  {
-    GCC_send_destroy (iter->c);
-  }
-  GCT_destroy (t);
-}
-
-
-/**
- * Tunnel is empty: destroy it.
- *
- * Notifies all connections about the destruction.
- *
- * @param t Tunnel to destroy.
- */
-void
-GCT_destroy_empty (struct CadetTunnel *t)
-{
-  if (GNUNET_YES == shutting_down)
-    return; /* Will be destroyed immediately anyway */
-
-  if (NULL != t->destroy_task)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Tunnel %s is already scheduled for destruction. Tunnel debug dump:\n",
-         GCT_2s (t));
-    GCT_debug (t, GNUNET_ERROR_TYPE_WARNING);
-    GNUNET_break (0);
-    /* should never happen, tunnel can only become empty once, and the
-     * task identifier should be NO_TASK (cleaned when the tunnel was created
-     * or became un-empty)
-     */
-    return;
-  }
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel %s empty: scheduling destruction\n",
-       GCT_2s (t));
-
-  // FIXME make delay a config option
-  t->destroy_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                                  &delayed_destroy, t);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Scheduled destroy of %p as %p\n",
-       t, t->destroy_task);
-}
-
-
-/**
- * Destroy tunnel if empty (no more channels).
- *
- * @param t Tunnel to destroy if empty.
- */
-void
-GCT_destroy_if_empty (struct CadetTunnel *t)
-{
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel %s destroy if empty\n", GCT_2s (t));
-  if (0 < GCT_count_channels (t))
-    return;
-
-  GCT_destroy_empty (t);
-}
-
-
-/**
- * Destroy the tunnel.
- *
- * This function does not generate any warning traffic to clients or peers.
- *
- * Tasks:
- * Cancel messages belonging to this tunnel queued to neighbors.
- * Free any allocated resources linked to the tunnel.
- *
- * @param t The tunnel to destroy.
- */
-void
-GCT_destroy (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter_c;
-  struct CadetTConnection *next_c;
-  struct CadetTChannel *iter_ch;
-  struct CadetTChannel *next_ch;
-  unsigned int keepalives_queued;
-
-  if (NULL == t)
-    return;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "destroying tunnel %s\n",
-       GCP_2s (t->peer));
-  GNUNET_break (GNUNET_YES ==
-                GNUNET_CONTAINER_multipeermap_remove (tunnels,
-                                                      GCP_get_id (t->peer), t));
-
-  for (iter_c = t->connection_head; NULL != iter_c; iter_c = next_c)
-  {
-    next_c = iter_c->next;
-    GCC_destroy (iter_c->c);
-  }
-  for (iter_ch = t->channel_head; NULL != iter_ch; iter_ch = next_ch)
-  {
-    next_ch = iter_ch->next;
-    GCCH_destroy (iter_ch->ch);
-    /* Should only happen on shutdown, but it's ok. */
-  }
-  keepalives_queued = 0;
-  while (NULL != t->tq_head)
-  {
-    /* Should have been cleaned by destuction of channel. */
-    struct GNUNET_MessageHeader *mh;
-    uint16_t type;
-
-    mh = (struct GNUNET_MessageHeader *) &t->tq_head[1];
-    type = ntohs (mh->type);
-    if (0 == keepalives_queued && GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE == type)
-    {
-      keepalives_queued = 1;
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "one keepalive left behind on tunnel shutdown\n");
-    }
-    else if (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY == type)
-    {
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           "tunnel destroyed before a CHANNEL_DESTROY was sent to peer\n");
-    }
-    else
-    {
-      GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           "message left behind on tunnel shutdown: %s\n",
-           GC_m2s (type));
-    }
-    unqueue_data (t->tq_head);
-  }
-
-
-  if (NULL != t->destroy_task)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "cancelling dest: %p\n",
-         t->destroy_task);
-    GNUNET_SCHEDULER_cancel (t->destroy_task);
-    t->destroy_task = NULL;
-  }
-
-  if (NULL != t->trim_connections_task)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "cancelling trim: %p\n",
-         t->trim_connections_task);
-    GNUNET_SCHEDULER_cancel (t->trim_connections_task);
-    t->trim_connections_task = NULL;
-  }
-
-  GNUNET_STATISTICS_update (stats, "# tunnels", -1, GNUNET_NO);
-  GCP_set_tunnel (t->peer, NULL);
-
-  if (NULL != t->rekey_task)
-  {
-    GNUNET_SCHEDULER_cancel (t->rekey_task);
-    t->rekey_task = NULL;
-  }
-  if (NULL != t->ax)
-    destroy_ax (t);
-
-  GNUNET_free (t);
-}
-
-
-/**
- * @brief Use the given path for the tunnel.
- * Update the next and prev hops (and RCs).
- * (Re)start the path refresh in case the tunnel is locally owned.
- *
- * @param t Tunnel to update.
- * @param p Path to use.
- *
- * @return Connection created.
- */
-struct CadetConnection *
-GCT_use_path (struct CadetTunnel *t, struct CadetPeerPath *path)
-{
-  struct CadetConnection *c;
-  struct GNUNET_CADET_ConnectionTunnelIdentifier cid;
-  unsigned int own_pos;
-
-  if (NULL == t || NULL == path)
-  {
-    GNUNET_break (0);
-    return NULL;
-  }
-
-  if (CADET_TUNNEL_SHUTDOWN == t->cstate)
-  {
-    GNUNET_break (0);
-    return NULL;
-  }
-
-  for (own_pos = 0; own_pos < path->length; own_pos++)
-  {
-    if (path->peers[own_pos] == myid)
-      break;
-  }
-  if (own_pos >= path->length)
-  {
-    GNUNET_break_op (0);
-    return NULL;
-  }
-
-  GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_NONCE, &cid, sizeof (cid));
-  c = GCC_new (&cid, t, path, own_pos);
-  if (NULL == c)
-  {
-    /* Path was flawed */
-    return NULL;
-  }
-  GCT_add_connection (t, c);
-  return c;
-}
-
-
-/**
- * Count all created connections of a tunnel. Not necessarily ready connections!
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of connections created, either being established or ready.
- */
-unsigned int
-GCT_count_any_connections (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter;
-  unsigned int count;
-
-  if (NULL == t)
-    return 0;
-
-  for (count = 0, iter = t->connection_head; NULL != iter; iter = iter->next)
-    count++;
-
-  return count;
-}
-
-
-/**
- * Count established (ready) connections of a tunnel.
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of connections.
- */
-unsigned int
-GCT_count_connections (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter;
-  unsigned int count;
-
-  if (NULL == t)
-    return 0;
-
-  for (count = 0, iter = t->connection_head; NULL != iter; iter = iter->next)
-    if (CADET_CONNECTION_READY == GCC_get_state (iter->c))
-      count++;
-
-  return count;
-}
-
-
-/**
- * Count channels of a tunnel.
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of channels.
- */
-unsigned int
-GCT_count_channels (struct CadetTunnel *t)
-{
-  struct CadetTChannel *iter;
-  unsigned int count;
-
-  for (count = 0, iter = t->channel_head;
-       NULL != iter;
-       iter = iter->next, count++) /* skip */;
-
-  return count;
-}
-
-
-/**
- * Get the connectivity state of a tunnel.
- *
- * @param t Tunnel.
- *
- * @return Tunnel's connectivity state.
- */
-enum CadetTunnelCState
-GCT_get_cstate (struct CadetTunnel *t)
-{
-  if (NULL == t)
-  {
-    GNUNET_assert (0);
-    return (enum CadetTunnelCState) -1;
-  }
-  return t->cstate;
-}
-
-
-/**
- * Get the encryption state of a tunnel.
- *
- * @param t Tunnel.
- *
- * @return Tunnel's encryption state.
- */
-enum CadetTunnelEState
-GCT_get_estate (struct CadetTunnel *t)
-{
-  if (NULL == t)
-  {
-    GNUNET_break (0);
-    return (enum CadetTunnelEState) -1;
-  }
-  return t->estate;
-}
-
-/**
- * Get the maximum buffer space for a tunnel towards a local client.
- *
- * @param t Tunnel.
- *
- * @return Biggest buffer space offered by any channel in the tunnel.
- */
-unsigned int
-GCT_get_channels_buffer (struct CadetTunnel *t)
-{
-  struct CadetTChannel *iter;
-  unsigned int buffer;
-  unsigned int ch_buf;
-
-  if (NULL == t->channel_head)
-  {
-    /* Probably getting buffer for a channel create/handshake. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  no channels, allow max\n");
-    return MIN_TUNNEL_BUFFER;
-  }
-
-  buffer = 0;
-  for (iter = t->channel_head; NULL != iter; iter = iter->next)
-  {
-    ch_buf = get_channel_buffer (iter);
-    if (ch_buf > buffer)
-      buffer = ch_buf;
-  }
-  if (MIN_TUNNEL_BUFFER > buffer)
-    return MIN_TUNNEL_BUFFER;
-
-  if (MAX_TUNNEL_BUFFER < buffer)
-  {
-    GNUNET_break (0);
-    return MAX_TUNNEL_BUFFER;
-  }
-  return buffer;
-}
-
-
-/**
- * Get the total buffer space for a tunnel for P2P traffic.
- *
- * @param t Tunnel.
- *
- * @return Buffer space offered by all connections in the tunnel.
- */
-unsigned int
-GCT_get_connections_buffer (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter;
-  unsigned int buffer;
-
-  if (GNUNET_NO == is_ready (t))
-  {
-    if (count_queued_data (t) >= 3)
-      return 0;
-    else
-      return 1;
-  }
-
-  buffer = 0;
-  for (iter = t->connection_head; NULL != iter; iter = iter->next)
-  {
-    if (GCC_get_state (iter->c) != CADET_CONNECTION_READY)
-    {
-      continue;
-    }
-    buffer += get_connection_buffer (iter);
-  }
-
-  return buffer;
-}
-
-
-/**
- * Get the tunnel's destination.
- *
- * @param t Tunnel.
- *
- * @return ID of the destination peer.
- */
-const struct GNUNET_PeerIdentity *
-GCT_get_destination (struct CadetTunnel *t)
-{
-  return GCP_get_id (t->peer);
-}
-
-
-/**
- * Get the tunnel's next free global channel ID.
- *
- * @param t Tunnel.
- *
- * @return GID of a channel free to use.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCT_get_next_ctn (struct CadetTunnel *t)
-{
-  struct GNUNET_CADET_ChannelTunnelNumber ctn;
-  struct GNUNET_CADET_ChannelTunnelNumber mask;
-  int result;
-
-  /* Set bit 30 depending on the ID relationship. Bit 31 is always 0 for GID.
-   * If our ID is bigger or loopback tunnel, start at 0, bit 30 = 0
-   * If peer's ID is bigger, start at 0x4... bit 30 = 1
-   */
-  result = GNUNET_CRYPTO_cmp_peer_identity (&my_full_id, GCP_get_id (t->peer));
-  if (0 > result)
-    mask.cn = htonl (0x40000000);
-  else
-    mask.cn = 0x0;
-  t->next_ctn.cn |= mask.cn;
-
-  while (NULL != GCT_get_channel (t, t->next_ctn))
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Channel %u exists...\n",
-         t->next_ctn.cn);
-    t->next_ctn.cn = htonl ((ntohl (t->next_ctn.cn) + 1) & ~GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
-    t->next_ctn.cn |= mask.cn;
-  }
-  ctn = t->next_ctn;
-  t->next_ctn.cn = (t->next_ctn.cn + 1) & ~GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
-  t->next_ctn.cn |= mask.cn;
-
-  return ctn;
-}
-
-
-/**
- * Send ACK on one or more channels due to buffer in connections.
- *
- * @param t Channel which has some free buffer space.
- */
-void
-GCT_unchoke_channels (struct CadetTunnel *t)
-{
-  struct CadetTChannel *iter;
-  unsigned int buffer;
-  unsigned int channels = GCT_count_channels (t);
-  unsigned int choked_n;
-  struct CadetChannel *choked[channels];
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GCT_unchoke_channels on %s\n", GCT_2s (t));
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " head: %p\n", t->channel_head);
-  if (NULL != t->channel_head)
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " head ch: %p\n", t->channel_head->ch);
-
-  if (NULL != t->tq_head)
-    send_queued_data (t);
-
-  /* Get buffer space */
-  buffer = GCT_get_connections_buffer (t);
-  if (0 == buffer)
-  {
-    return;
-  }
-
-  /* Count and remember choked channels */
-  choked_n = 0;
-  for (iter = t->channel_head; NULL != iter; iter = iter->next)
-  {
-    if (GNUNET_NO == get_channel_allowed (iter))
-    {
-      choked[choked_n++] = iter->ch;
-    }
-  }
-
-  /* Unchoke random channels */
-  while (0 < buffer && 0 < choked_n)
-  {
-    unsigned int r = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                               choked_n);
-    GCCH_allow_client (choked[r], GCCH_is_origin (choked[r], GNUNET_YES));
-    choked_n--;
-    buffer--;
-    choked[r] = choked[choked_n];
-  }
-}
-
-
-/**
- * Send ACK on one or more connections due to buffer space to the client.
- *
- * Iterates all connections of the tunnel and sends ACKs appropriately.
- *
- * @param t Tunnel.
- */
-void
-GCT_send_connection_acks (struct CadetTunnel *t)
-{
-  struct CadetTConnection *iter;
-  uint32_t allowed;
-  uint32_t to_allow;
-  uint32_t allow_per_connection;
-  unsigned int cs;
-  unsigned int buffer;
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel send connection ACKs on %s\n",
-       GCT_2s (t));
-
-  if (NULL == t)
-  {
-    GNUNET_break (0);
-    return;
-  }
-
-  if (CADET_TUNNEL_READY != t->cstate)
-    return;
-
-  buffer = GCT_get_channels_buffer (t);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  buffer %u\n", buffer);
-
-  /* Count connections, how many messages are already allowed */
-  cs = GCT_count_connections (t);
-  for (allowed = 0, iter = t->connection_head; NULL != iter; iter = iter->next)
-  {
-    allowed += get_connection_allowed (iter);
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  allowed %u\n", allowed);
-
-  /* Make sure there is no overflow */
-  if (allowed > buffer)
-    return;
-
-  /* Authorize connections to send more data */
-  to_allow = buffer - allowed;
-
-  for (iter = t->connection_head;
-       NULL != iter && to_allow > 0;
-       iter = iter->next)
-  {
-    if (CADET_CONNECTION_READY != GCC_get_state (iter->c)
-        || get_connection_allowed (iter) > 64 / 3)
-    {
-      continue;
-    }
-    GNUNET_assert(cs != 0);
-    allow_per_connection = to_allow/cs;
-    to_allow -= allow_per_connection;
-    cs--;
-    GCC_allow (iter->c, allow_per_connection,
-               GCC_is_origin (iter->c, GNUNET_NO));
-  }
-
-  if (0 != to_allow)
-  {
-    /* Since we don't allow if it's allowed to send 64/3, this can happen. */
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  reminding to_allow: %u\n", to_allow);
-  }
-}
-
-
-/**
- * Cancel a previously sent message while it's in the queue.
- *
- * ONLY can be called before the continuation given to the send function
- * is called. Once the continuation is called, the message is no longer in the
- * queue.
- *
- * @param q Handle to the queue.
- */
-void
-GCT_cancel (struct CadetTunnelQueue *q)
-{
-  if (NULL != q->cq)
-  {
-    GNUNET_assert (NULL == q->tqd);
-    GCC_cancel (q->cq);
-    /* tun_message_sent() will be called and free q */
-  }
-  else if (NULL != q->tqd)
-  {
-    unqueue_data (q->tqd);
-    q->tqd = NULL;
-    if (NULL != q->cont)
-      q->cont (q->cont_cls, NULL, q, 0, 0);
-    GNUNET_free (q);
-  }
-  else
-  {
-    GNUNET_break (0);
-  }
-}
-
-
-/**
- * Check if the tunnel has queued traffic.
- *
- * @param t Tunnel to check.
- *
- * @return #GNUNET_YES if there is queued traffic
- *         #GNUNET_NO otherwise
- */
-int
-GCT_has_queued_traffic (struct CadetTunnel *t)
-{
-  return (NULL != t->tq_head) ? GNUNET_YES : GNUNET_NO;
-}
-
-
-/**
- * Sends an already built message on a tunnel, encrypting it and
- * choosing the best connection if not provided.
- *
- * @param message Message to send. Function modifies it.
- * @param t Tunnel on which this message is transmitted.
- * @param c Connection to use (autoselect if NULL).
- * @param force Force the tunnel to take the message (buffer overfill).
- * @param cont Continuation to call once message is really sent.
- * @param cont_cls Closure for @c cont.
- *
- * @return Handle to cancel message. NULL if @c cont is NULL.
- */
-struct CadetTunnelQueue *
-GCT_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                           struct CadetTunnel *t,
-                           struct CadetConnection *c,
-                           int force, GCT_sent cont, void *cont_cls)
-{
-  return send_prebuilt_message (message, t, c, force, cont, cont_cls, NULL);
-}
-
-
-/**
- * Send a KX message.
- *
- * @param t Tunnel on which to send it.
- * @param force_reply Force the other peer to reply with a KX message.
- */
-void
-GCT_send_kx (struct CadetTunnel *t, int force_reply)
-{
-  static struct CadetEncryptedMessageIdentifier zero;
-  struct CadetConnection *c;
-  struct GNUNET_CADET_TunnelKeyExchangeMessage msg;
-  enum GNUNET_CADET_KX_Flags flags;
-
-  LOG (GNUNET_ERROR_TYPE_INFO, "==> {        KX} on %s\n", GCT_2s (t));
-  if (NULL != t->ephm_h)
-  {
-    LOG (GNUNET_ERROR_TYPE_INFO, "     already queued, nop\n");
-    return;
-  }
-  GNUNET_assert (GNUNET_NO == GCT_is_loopback (t));
-
-  c = tunnel_get_connection (t);
-  if (NULL == c)
-  {
-    if (NULL == t->destroy_task && CADET_TUNNEL_READY == t->cstate)
-    {
-      GNUNET_break (0);
-      GCT_debug (t, GNUNET_ERROR_TYPE_ERROR);
-    }
-    return;
-  }
-
-  msg.header.size = htons (sizeof (msg));
-  msg.header.type = htons (GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX);
-  flags = GNUNET_CADET_KX_FLAG_NONE;
-  if (GNUNET_YES == force_reply)
-    flags |= GNUNET_CADET_KX_FLAG_FORCE_REPLY;
-  msg.flags = htonl (flags);
-  msg.cid = *GCC_get_id (c);
-  GNUNET_CRYPTO_ecdhe_key_get_public (t->ax->kx_0, &msg.ephemeral_key);
-  GNUNET_CRYPTO_ecdhe_key_get_public (t->ax->DHRs, &msg.ratchet_key);
-
-  t->ephm_h = GCC_send_prebuilt_message (&msg.header,
-                                         UINT16_MAX,
-                                         zero,
-                                         c,
-                                         GCC_is_origin (c, GNUNET_YES),
-                                         GNUNET_YES, &ephm_sent, t);
-  if (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate)
-    GCT_change_estate (t, CADET_TUNNEL_KEY_AX_SENT);
-}
-
-
-/**
- * Is the tunnel directed towards the local peer?
- *
- * @param t Tunnel.
- *
- * @return #GNUNET_YES if it is loopback.
- */
-int
-GCT_is_loopback (const struct CadetTunnel *t)
-{
-  return (myid == GCP_get_short_id (t->peer));
-}
-
-
-/**
- * Is the tunnel this path already?
- *
- * @param t Tunnel.
- * @param p Path.
- *
- * @return #GNUNET_YES a connection uses this path.
- */
-int
-GCT_is_path_used (const struct CadetTunnel *t, const struct CadetPeerPath *p)
-{
-  struct CadetTConnection *iter;
-
-  for (iter = t->connection_head; NULL != iter; iter = iter->next)
-    if (path_equivalent (GCC_get_path (iter->c), p))
-      return GNUNET_YES;
-
-  return GNUNET_NO;
-}
-
-
-/**
- * Get a cost of a path for a tunnel considering existing connections.
- *
- * @param t Tunnel.
- * @param path Candidate path.
- *
- * @return Cost of the path (path length + number of overlapping nodes)
- */
-unsigned int
-GCT_get_path_cost (const struct CadetTunnel *t,
-                   const struct CadetPeerPath *path)
-{
-  struct CadetTConnection *iter;
-  const struct CadetPeerPath *aux;
-  unsigned int overlap;
-  unsigned int i;
-  unsigned int j;
-
-  if (NULL == path)
-    return 0;
-
-  overlap = 0;
-  GNUNET_assert (NULL != t);
-
-  for (i = 0; i < path->length; i++)
-  {
-    for (iter = t->connection_head; NULL != iter; iter = iter->next)
-    {
-      aux = GCC_get_path (iter->c);
-      if (NULL == aux)
-        continue;
-
-      for (j = 0; j < aux->length; j++)
-      {
-        if (path->peers[i] == aux->peers[j])
-        {
-          overlap++;
-          break;
-        }
-      }
-    }
-  }
-  return path->length + overlap;
-}
-
-
-/**
- * Get the static string for the peer this tunnel is directed.
- *
- * @param t Tunnel.
- *
- * @return Static string the destination peer's ID.
- */
-const char *
-GCT_2s (const struct CadetTunnel *t)
-{
-  if (NULL == t)
-    return "(NULL)";
-
-  return GCP_2s (t->peer);
-}
-
-
-/******************************************************************************/
-/*****************************    INFO/DEBUG    *******************************/
-/******************************************************************************/
-
-static void
-ax_debug (const struct CadetTunnelAxolotl *ax, enum GNUNET_ErrorType level)
-{
-  struct GNUNET_CRYPTO_EcdhePublicKey pub;
-  struct CadetTunnelSkippedKey *iter;
-
-  LOG2 (level, "TTT  RK  \t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->RK));
-
-  LOG2 (level, "TTT  HKs \t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->HKs));
-  LOG2 (level, "TTT  HKr \t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->HKr));
-  LOG2 (level, "TTT  NHKs\t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->NHKs));
-  LOG2 (level, "TTT  NHKr\t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->NHKr));
-
-  LOG2 (level, "TTT  CKs \t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->CKs));
-  LOG2 (level, "TTT  CKr \t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->CKr));
-
-  GNUNET_CRYPTO_ecdhe_key_get_public (ax->DHRs, &pub);
-  LOG2 (level, "TTT  DHRs\t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &pub));
-  LOG2 (level, "TTT  DHRr\t %s\n",
-        GNUNET_i2s ((struct GNUNET_PeerIdentity *) &ax->DHRr));
-
-  LOG2 (level, "TTT  Nr\t %u\tNs\t%u\n", ax->Nr, ax->Ns);
-  LOG2 (level, "TTT  PNs\t %u\tSkipped\t%u\n", ax->PNs, ax->skipped);
-  LOG2 (level, "TTT  Ratchet\t%u\n", ax->ratchet_flag);
-
-  for (iter = ax->skipped_head; NULL != iter; iter = iter->next)
-  {
-    LOG2 (level, "TTT    HK\t %s\n",
-          GNUNET_i2s ((struct GNUNET_PeerIdentity *) &iter->HK));
-    LOG2 (level, "TTT    MK\t %s\n",
-          GNUNET_i2s ((struct GNUNET_PeerIdentity *) &iter->MK));
-  }
-}
-
-/**
- * Log all possible info about the tunnel state.
- *
- * @param t Tunnel to debug.
- * @param level Debug level to use.
- */
-void
-GCT_debug (const struct CadetTunnel *t, enum GNUNET_ErrorType level)
-{
-  struct CadetTChannel *iter_ch;
-  struct CadetTConnection *iter_c;
-  int do_log;
-
-  do_log = GNUNET_get_log_call_status (level & (~GNUNET_ERROR_TYPE_BULK),
-                                       "cadet-tun",
-                                       __FILE__, __FUNCTION__, __LINE__);
-  if (0 == do_log)
-    return;
-
-  LOG2 (level, "TTT DEBUG TUNNEL TOWARDS %s\n", GCT_2s (t));
-  LOG2 (level, "TTT  cstate %s, estate %s\n",
-       cstate2s (t->cstate), estate2s (t->estate));
-#if DUMP_KEYS_TO_STDERR
-  ax_debug (t->ax, level);
-#endif
-  LOG2 (level, "TTT  tq_head %p, tq_tail %p\n", t->tq_head, t->tq_tail);
-  LOG2 (level, "TTT  destroy %p\n", t->destroy_task);
-  LOG2 (level, "TTT  channels:\n");
-  for (iter_ch = t->channel_head; NULL != iter_ch; iter_ch = iter_ch->next)
-  {
-    GCCH_debug (iter_ch->ch, level);
-  }
-
-  LOG2 (level, "TTT  connections:\n");
-  for (iter_c = t->connection_head; NULL != iter_c; iter_c = iter_c->next)
-  {
-    GCC_debug (iter_c->c, level);
-  }
-
-  LOG2 (level, "TTT DEBUG TUNNEL END\n");
-}
-
-
-/**
- * Iterate all tunnels.
- *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter, void *cls)
-{
-  GNUNET_CONTAINER_multipeermap_iterate (tunnels, iter, cls);
-}
-
-
-/**
- * Count all tunnels.
- *
- * @return Number of tunnels to remote peers kept by this peer.
- */
-unsigned int
-GCT_count_all (void)
-{
-  return GNUNET_CONTAINER_multipeermap_size (tunnels);
-}
-
-
-/**
- * Iterate all connections of a tunnel.
- *
- * @param t Tunnel whose connections to iterate.
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_connections (struct CadetTunnel *t, GCT_conn_iter iter, void *cls)
-{
-  struct CadetTConnection *ct;
-
-  for (ct = t->connection_head; NULL != ct; ct = ct->next)
-    iter (cls, ct->c);
-}
-
-
-/**
- * Iterate all channels of a tunnel.
- *
- * @param t Tunnel whose channels to iterate.
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_channels (struct CadetTunnel *t, GCT_chan_iter iter, void *cls)
-{
-  struct CadetTChannel *cht;
-
-  for (cht = t->channel_head; NULL != cht; cht = cht->next)
-    iter (cls, cht->ch);
-}
diff --git a/src/cadet/gnunet-service-cadet_tunnel.h b/src/cadet/gnunet-service-cadet_tunnel.h
deleted file mode 100644
index 1b56a0632..000000000
--- a/src/cadet/gnunet-service-cadet_tunnel.h
+++ /dev/null
@@ -1,616 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/gnunet-service-cadet_tunnel.h
- * @brief cadet service; dealing with tunnels and crypto
- * @author Bartlomiej Polot
- *
- * All functions in this file should use the prefix GMT (Gnunet Cadet Tunnel)
- */
-
-#ifndef GNUNET_SERVICE_CADET_TUNNEL_H
-#define GNUNET_SERVICE_CADET_TUNNEL_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define CONNECTIONS_PER_TUNNEL 3
-
-/**
- * All the connectivity states a tunnel can be in.
- */
-enum CadetTunnelCState
-{
-  /**
-   * Uninitialized status, should never appear in operation.
-   */
-  CADET_TUNNEL_NEW,
-
-  /**
-   * No path to the peer known yet.
-   */
-  CADET_TUNNEL_SEARCHING,
-
-  /**
-   * Request sent, not yet answered.
-   */
-  CADET_TUNNEL_WAITING,
-
-  /**
-   * Peer connected and ready to accept data.
-   */
-  CADET_TUNNEL_READY,
-
-  /**
-   * Tunnel being shut down, don't try to keep it alive.
-   */
-  CADET_TUNNEL_SHUTDOWN
-};
-
-
-/**
- * All the encryption states a tunnel can be in.
- */
-enum CadetTunnelEState
-{
-  /**
-   * Uninitialized status, should never appear in operation.
-   */
-  CADET_TUNNEL_KEY_UNINITIALIZED,
-
-  /**
-   * Ephemeral key sent, waiting for peer's key.
-   */
-  CADET_TUNNEL_KEY_AX_SENT,
-
-  /**
-   * In OTR: New ephemeral key and ping sent, waiting for pong.
-   *
-   * This means that we DO have the peer's ephemeral key, otherwise the
-   * state would be KEY_SENT. We DO NOT have a valid session key (either no
-   * previous key or previous key expired).
-   *
-   *
-   * In Axolotl: Key sent and received but no deciphered traffic yet.
-   *
-   * This means that we can send traffic (otherwise we would never complete
-   * the handshake), but we don't have complete confirmation. Since the first
-   * traffic MUST be a complete channel creation 3-way handshake, no payload
-   * will be sent before confirmation.
-   */
-  CADET_TUNNEL_KEY_AX_AUTH_SENT,
-
-  /**
-   * Handshake completed: session key available.
-   */
-  CADET_TUNNEL_KEY_OK,
-
-  /**
-   * New ephemeral key and ping sent, waiting for pong. Unlike KEY_PING,
-   * we still have a valid session key and therefore we *can* still send
-   * traffic on the tunnel.
-   */
-  CADET_TUNNEL_KEY_REKEY
-};
-
-/**
- * Struct containing all information regarding a given peer
- */
-struct CadetTunnel;
-
-
-#include "gnunet-service-cadet_channel.h"
-#include "gnunet-service-cadet_connection.h"
-#include "gnunet-service-cadet_peer.h"
-
-/**
- * Handle for messages queued but not yet sent.
- */
-struct CadetTunnelQueue;
-
-/**
- * Callback called when a queued message is sent.
- *
- * @param cls Closure.
- * @param t Tunnel this message was on.
- * @param type Type of message sent.
- * @param size Size of the message.
- */
-typedef void
-(*GCT_sent) (void *cls,
-             struct CadetTunnel *t,
-             struct CadetTunnelQueue *q,
-             uint16_t type, size_t size);
-
-typedef void
-(*GCT_conn_iter) (void *cls, struct CadetConnection *c);
-
-
-typedef void
-(*GCT_chan_iter) (void *cls, struct CadetChannel *ch);
-
-
-/******************************************************************************/
-/********************************    API    ***********************************/
-/******************************************************************************/
-
-/**
- * Initialize tunnel subsystem.
- *
- * @param c Configuration handle.
- * @param key ECC private key, to derive all other keys and do crypto.
- */
-void
-GCT_init (const struct GNUNET_CONFIGURATION_Handle *c,
-          const struct GNUNET_CRYPTO_EddsaPrivateKey *key);
-
-
-/**
- * Shut down the tunnel subsystem.
- */
-void
-GCT_shutdown (void);
-
-
-/**
- * Create a tunnel.
- *
- * @param destination Peer this tunnel is towards.
- */
-struct CadetTunnel *
-GCT_new (struct CadetPeer *destination);
-
-
-/**
- * Tunnel is empty: destroy it.
- *
- * Notifies all connections about the destruction.
- *
- * @param t Tunnel to destroy.
- */
-void
-GCT_destroy_empty (struct CadetTunnel *t);
-
-
-/**
- * Destroy tunnel if empty (no more channels).
- *
- * @param t Tunnel to destroy if empty.
- */
-void
-GCT_destroy_if_empty (struct CadetTunnel *t);
-
-
-/**
- * Destroy the tunnel.
- *
- * This function does not generate any warning traffic to clients or peers.
- *
- * Tasks:
- * Cancel messages belonging to this tunnel queued to neighbors.
- * Free any allocated resources linked to the tunnel.
- *
- * @param t The tunnel to destroy.
- */
-void
-GCT_destroy (struct CadetTunnel *t);
-
-
-/**
- * Change the tunnel's connection state.
- *
- * @param t Tunnel whose connection state to change.
- * @param cstate New connection state.
- */
-void
-GCT_change_cstate (struct CadetTunnel* t, enum CadetTunnelCState cstate);
-
-
-/**
- * Change the tunnel encryption state.
- *
- * @param t Tunnel whose encryption state to change.
- * @param state New encryption state.
- */
-void
-GCT_change_estate (struct CadetTunnel* t, enum CadetTunnelEState state);
-
-
-/**
- * Add a connection to a tunnel.
- *
- * @param t Tunnel.
- * @param c Connection.
- */
-void
-GCT_add_connection (struct CadetTunnel *t, struct CadetConnection *c);
-
-
-/**
- * Remove a connection from a tunnel.
- *
- * @param t Tunnel.
- * @param c Connection.
- */
-void
-GCT_remove_connection (struct CadetTunnel *t, struct CadetConnection *c);
-
-
-/**
- * Add a channel to a tunnel.
- *
- * @param t Tunnel.
- * @param ch Channel.
- */
-void
-GCT_add_channel (struct CadetTunnel *t, struct CadetChannel *ch);
-
-
-/**
- * Remove a channel from a tunnel.
- *
- * @param t Tunnel.
- * @param ch Channel.
- */
-void
-GCT_remove_channel (struct CadetTunnel *t, struct CadetChannel *ch);
-
-
-/**
- * Search for a channel by global ID.
- *
- * @param t Tunnel containing the channel.
- * @param ctn Public channel number.
- *
- * @return channel handler, NULL if doesn't exist
- */
-struct CadetChannel *
-GCT_get_channel (struct CadetTunnel *t, struct GNUNET_CADET_ChannelTunnelNumber ctn);
-
-
-/**
- * Decrypt and process an encrypted message.
- *
- * Calls the appropriate handler for a message in a channel of a local tunnel.
- *
- * @param t Tunnel this message came on.
- * @param msg Message header.
- */
-void
-GCT_handle_encrypted (struct CadetTunnel *t,
-                      const struct GNUNET_CADET_TunnelEncryptedMessage *msg);
-
-
-/**
- * Handle a Key eXchange message.
- *
- * @param t Tunnel on which the message came.
- * @param msg KX message itself.
- */
-void
-GCT_handle_kx (struct CadetTunnel *t,
-               const struct GNUNET_CADET_TunnelKeyExchangeMessage *msg);
-
-
-/**
- * @brief Use the given path for the tunnel.
- * Update the next and prev hops (and RCs).
- * (Re)start the path refresh in case the tunnel is locally owned.
- *
- * @param t Tunnel to update.
- * @param p Path to use.
- *
- * @return Connection created.
- */
-struct CadetConnection *
-GCT_use_path (struct CadetTunnel *t, struct CadetPeerPath *p);
-
-
-/**
- * Count all created connections of a tunnel. Not necessarily ready connections!
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of connections created, either being established or ready.
- */
-unsigned int
-GCT_count_any_connections (struct CadetTunnel *t);
-
-
-/**
- * Count established (ready) connections of a tunnel.
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of connections.
- */
-unsigned int
-GCT_count_connections (struct CadetTunnel *t);
-
-
-/**
- * Count channels of a tunnel.
- *
- * @param t Tunnel on which to count.
- *
- * @return Number of channels.
- */
-unsigned int
-GCT_count_channels (struct CadetTunnel *t);
-
-
-/**
- * Get the connectivity state of a tunnel.
- *
- * @param t Tunnel.
- *
- * @return Tunnel's connectivity state.
- */
-enum CadetTunnelCState
-GCT_get_cstate (struct CadetTunnel *t);
-
-
-/**
- * Get the encryption state of a tunnel.
- *
- * @param t Tunnel.
- *
- * @return Tunnel's encryption state.
- */
-enum CadetTunnelEState
-GCT_get_estate (struct CadetTunnel *t);
-
-
-/**
- * Get the maximum buffer space for a tunnel towards a local client.
- *
- * @param t Tunnel.
- *
- * @return Biggest buffer space offered by any channel in the tunnel.
- */
-unsigned int
-GCT_get_channels_buffer (struct CadetTunnel *t);
-
-
-/**
- * Get the total buffer space for a tunnel for P2P traffic.
- *
- * @param t Tunnel.
- *
- * @return Buffer space offered by all connections in the tunnel.
- */
-unsigned int
-GCT_get_connections_buffer (struct CadetTunnel *t);
-
-
-/**
- * Get the tunnel's destination.
- *
- * @param t Tunnel.
- *
- * @return ID of the destination peer.
- */
-const struct GNUNET_PeerIdentity *
-GCT_get_destination (struct CadetTunnel *t);
-
-
-/**
- * Get the tunnel's next free Channel ID.
- *
- * @param t Tunnel.
- *
- * @return ID of a channel free to use.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-GCT_get_next_ctn (struct CadetTunnel *t);
-
-
-/**
- * Send ACK on one or more channels due to buffer in connections.
- *
- * @param t Channel which has some free buffer space.
- */
-void
-GCT_unchoke_channels (struct CadetTunnel *t);
-
-
-/**
- * Send ACK on one or more connections due to buffer space to the client.
- *
- * Iterates all connections of the tunnel and sends ACKs appropriately.
- *
- * @param t Tunnel which has some free buffer space.
- */
-void
-GCT_send_connection_acks (struct CadetTunnel *t);
-
-
-/**
- * Cancel a previously sent message while it's in the queue.
- *
- * ONLY can be called before the continuation given to the send function
- * is called. Once the continuation is called, the message is no longer in the
- * queue.
- *
- * @param q Handle to the queue.
- */
-void
-GCT_cancel (struct CadetTunnelQueue *q);
-
-
-/**
- * Check if the tunnel has queued traffic.
- *
- * @param t Tunnel to check.
- *
- * @return #GNUNET_YES if there is queued traffic
- *         #GNUNET_NO otherwise
- */
-int
-GCT_has_queued_traffic (struct CadetTunnel *t);
-
-/**
- * Sends an already built message on a tunnel, encrypting it and
- * choosing the best connection.
- *
- * @param message Message to send. Function modifies it.
- * @param t Tunnel on which this message is transmitted.
- * @param c Connection to use (autoselect if NULL).
- * @param force Force the tunnel to take the message (buffer overfill).
- * @param cont Continuation to call once message is really sent.
- * @param cont_cls Closure for @c cont.
- *
- * @return Handle to cancel message. NULL if @c cont is NULL.
- */
-struct CadetTunnelQueue *
-GCT_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                           struct CadetTunnel *t, struct CadetConnection *c,
-                           int force, GCT_sent cont, void *cont_cls);
-
-
-/**
- * Send a KX message.
- *
- * @param t Tunnel on which to send it.
- * @param force_reply Force the other peer to reply with a KX message.
- */
-void
-GCT_send_kx (struct CadetTunnel *t, int force_reply);
-
-
-/**
- * Is the tunnel directed towards the local peer?
- *
- * @param t Tunnel.
- *
- * @return #GNUNET_YES if it is loopback.
- */
-int
-GCT_is_loopback (const struct CadetTunnel *t);
-
-
-/**
- * Is the tunnel using this path already?
- *
- * @param t Tunnel.
- * @param p Path.
- *
- * @return #GNUNET_YES a connection uses this path.
- */
-int
-GCT_is_path_used (const struct CadetTunnel *t, const struct CadetPeerPath *p);
-
-
-/**
- * Get a cost of a path for a tunnel considering existing connections.
- *
- * @param t Tunnel.
- * @param path Candidate path.
- *
- * @return Cost of the path (path length + number of overlapping nodes)
- */
-unsigned int
-GCT_get_path_cost (const struct CadetTunnel *t,
-                   const struct CadetPeerPath *path);
-
-
-/**
- * Get the static string for the peer this tunnel is directed.
- *
- * @param t Tunnel.
- *
- * @return Static string the destination peer's ID.
- */
-const char *
-GCT_2s (const struct CadetTunnel *t);
-
-
-/**
- * Log all possible info about the tunnel state.
- *
- * @param t Tunnel to debug.
- * @param level Debug level to use.
- */
-void
-GCT_debug (const struct CadetTunnel *t, enum GNUNET_ErrorType level);
-
-
-/**
- * Iterate all tunnels.
- *
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_all (GNUNET_CONTAINER_PeerMapIterator iter, void *cls);
-
-
-/**
- * Count all tunnels.
- *
- * @return Number of tunnels to remote peers kept by this peer.
- */
-unsigned int
-GCT_count_all (void);
-
-
-/**
- * Iterate all connections of a tunnel.
- *
- * @param t Tunnel whose connections to iterate.
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_connections (struct CadetTunnel *t, GCT_conn_iter iter, void *cls);
-
-
-/**
- * Iterate all channels of a tunnel.
- *
- * @param t Tunnel whose channels to iterate.
- * @param iter Iterator.
- * @param cls Closure for @c iter.
- */
-void
-GCT_iterate_channels (struct CadetTunnel *t,
-                      GCT_chan_iter iter,
-                      void *cls);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_CADET_SERVICE_TUNNEL_H */
-#endif
-/* end of gnunet-cadet-service_tunnel.h */
diff --git a/src/cadet/gnunet-service-cadet-new_tunnels.c b/src/cadet/gnunet-service-cadet_tunnels.c
index bf05fae6b..bcdeeb4da 100644
--- a/src/cadet/gnunet-service-cadet-new_tunnels.c
+++ b/src/cadet/gnunet-service-cadet_tunnels.c
@@ -18,7 +18,7 @@
      Boston, MA 02110-1301, USA.
 */
 /**
- * @file cadet/gnunet-service-cadet-new_tunnels.c
+ * @file cadet/gnunet-service-cadet_tunnels.c
  * @brief Information we track per tunnel.
  * @author Bartlomiej Polot
  * @author Christian Grothoff
@@ -34,13 +34,12 @@
 #include "gnunet_util_lib.h"
 #include "gnunet_statistics_service.h"
 #include "gnunet_signatures.h"
-#include "gnunet-service-cadet-new.h"
 #include "cadet_protocol.h"
-#include "gnunet-service-cadet-new_channel.h"
-#include "gnunet-service-cadet-new_connection.h"
-#include "gnunet-service-cadet-new_tunnels.h"
-#include "gnunet-service-cadet-new_peer.h"
-#include "gnunet-service-cadet-new_paths.h"
+#include "gnunet-service-cadet_channel.h"
+#include "gnunet-service-cadet_connection.h"
+#include "gnunet-service-cadet_tunnels.h"
+#include "gnunet-service-cadet_peer.h"
+#include "gnunet-service-cadet_paths.h"
 
 
 #define LOG(level, ...) GNUNET_log_from(level,"cadet-tun",__VA_ARGS__)
@@ -3082,10 +3081,6 @@ GCT_handle_encrypted (struct CadetTConnection *ct,
     break;
   }
 
-  GNUNET_STATISTICS_update (stats,
-                            "# received encrypted",
-                            1,
-                            GNUNET_NO);
   decrypted_size = -1;
   if (CADET_TUNNEL_KEY_OK == t->estate)
   {
@@ -3166,6 +3161,10 @@ GCT_handle_encrypted (struct CadetTConnection *ct,
              &t->ax);
     return;
   }
+  GNUNET_STATISTICS_update (stats,
+                            "# decrypted bytes",
+                            decrypted_size,
+                            GNUNET_NO);
 
   /* The MST will ultimately call #handle_decrypted() on each message. */
   t->current_ct = ct;
@@ -3217,6 +3216,10 @@ GCT_send (struct CadetTunnel *t,
                 &ax_msg[1],
                 message,
                 payload_size);
+  GNUNET_STATISTICS_update (stats,
+                            "# encrypted bytes",
+                            payload_size,
+                            GNUNET_NO);
   ax_msg->ax_header.Ns = htonl (t->ax.Ns++);
   ax_msg->ax_header.PNs = htonl (t->ax.PNs);
   /* FIXME: we should do this once, not once per message;
diff --git a/src/cadet/gnunet-service-cadet-new_tunnels.h b/src/cadet/gnunet-service-cadet_tunnels.h
index a81bc2341..4a3619ab6 100644
--- a/src/cadet/gnunet-service-cadet-new_tunnels.h
+++ b/src/cadet/gnunet-service-cadet_tunnels.h
@@ -20,7 +20,7 @@
 */
 
 /**
- * @file cadet/gnunet-service-cadet-new_tunnels.h
+ * @file cadet/gnunet-service-cadet_tunnels.h
  * @brief Information we track per tunnel.
  * @author Bartlomiej Polot
  * @author Christian Grothoff
@@ -28,7 +28,7 @@
 #ifndef GNUNET_SERVICE_CADET_TUNNELS_H
 #define GNUNET_SERVICE_CADET_TUNNELS_H
 
-#include "gnunet-service-cadet-new.h"
+#include "gnunet-service-cadet.h"
 #include "cadet_protocol.h"
 
 
diff --git a/src/cadet/test_cadet.c b/src/cadet/test_cadet.c
index e57c01be2..8963d97c9 100644
--- a/src/cadet/test_cadet.c
+++ b/src/cadet/test_cadet.c
@@ -21,7 +21,7 @@
  * @file cadet/test_cadet.c
  * @author Bart Polot
  * @author Christian Grothoff
- * @brief Test for the cadet service: retransmission of traffic.
+ * @brief Test for the cadet service using mq API.
  */
 #include <stdio.h>
 #include "platform.h"
@@ -32,9 +32,20 @@
 
 
 /**
- * How many messages to send
+ * Ugly workaround to unify data handlers on incoming and outgoing channels.
  */
-#define TOTAL_PACKETS 500 /* Cannot exceed 64k! */
+struct CadetTestChannelWrapper
+{
+  /**
+   * Channel pointer.
+   */
+  struct GNUNET_CADET_Channel *ch;
+};
+
+/**
+ * How many messages to send by default.
+ */
+#define TOTAL_PACKETS 500       /* Cannot exceed 64k! */
 
 /**
  * How long until we give up on connecting the peers?
@@ -42,7 +53,7 @@
 #define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 120)
 
 /**
- * Time to wait for stuff that should be rather fast
+ * Time to wait by default  for stuff that should be rather fast.
  */
 #define SHORT_TIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 20)
 
@@ -73,6 +84,16 @@ static char *test_name;
 static int test_backwards = GNUNET_NO;
 
 /**
+ * How many packets to send.
+ */
+static unsigned int total_packets;
+
+/**
+ * Time to wait for fast operations.
+ */
+static struct GNUNET_TIME_Relative short_time;
+
+/**
  * How many events have happened
  */
 static int ok;
@@ -83,9 +104,9 @@ static int ok;
 static int ok_goal;
 
 /**
- * Size of each test packet
+ * Size of each test packet's payload
  */
-static size_t size_payload = sizeof (struct GNUNET_MessageHeader) + sizeof (uint32_t);
+static size_t size_payload = sizeof (uint32_t);
 
 /**
  * Operation to get peer ids.
@@ -158,9 +179,9 @@ static struct GNUNET_SCHEDULER_Task *disconnect_task;
 static struct GNUNET_SCHEDULER_Task *test_task;
 
 /**
- * Task runnining #data_task().
+ * Task runnining #send_next_msg().
  */
-static struct GNUNET_SCHEDULER_Task *data_job;
+static struct GNUNET_SCHEDULER_Task *send_next_msg_task;
 
 /**
  * Cadet handle for the root peer
@@ -175,7 +196,7 @@ static struct GNUNET_CADET_Handle *h2;
 /**
  * Channel handle for the root peer
  */
-static struct GNUNET_CADET_Channel *ch;
+static struct GNUNET_CADET_Channel *outgoing_ch;
 
 /**
  * Channel handle for the dest peer
@@ -183,17 +204,6 @@ static struct GNUNET_CADET_Channel *ch;
 static struct GNUNET_CADET_Channel *incoming_ch;
 
 /**
- * Transmit handle for root data calls
- */
-static struct GNUNET_CADET_TransmitHandle *th;
-
-/**
- * Transmit handle for root data calls
- */
-static struct GNUNET_CADET_TransmitHandle *incoming_th;
-
-
-/**
  * Time we started the data transmission (after channel has been established
  * and initilized).
  */
@@ -225,20 +235,26 @@ static unsigned int ka_received;
 static unsigned int msg_dropped;
 
 
+/******************************************************************************/
+
+
+/******************************************************************************/
+
+
 /**
- * Get the client number considered as the "target" or "receiver", depending on
+ * Get the channel considered as the "target" or "receiver", depending on
  * the test type and size.
  *
- * @return Peer # of the target client, either 0 (for backward tests) or
- *         the last peer in the line (for other tests).
+ * @return Channel handle of the target client, either 0 (for backward tests)
+ *         or the last peer in the line (for other tests).
  */
-static unsigned int
-get_expected_target ()
+static struct GNUNET_CADET_Channel *
+get_target_channel ()
 {
   if (SPEED == test && GNUNET_YES == test_backwards)
-    return 0;
+    return outgoing_ch;
   else
-    return peers_requested - 1;
+    return incoming_ch;
 }
 
 
@@ -251,18 +267,15 @@ show_end_data (void)
   static struct GNUNET_TIME_Absolute end_time;
   static struct GNUNET_TIME_Relative total_time;
 
-  end_time = GNUNET_TIME_absolute_get();
-  total_time = GNUNET_TIME_absolute_get_difference(start_time, end_time);
+  end_time = GNUNET_TIME_absolute_get ();
+  total_time = GNUNET_TIME_absolute_get_difference (start_time, end_time);
   FPRINTF (stderr, "\nResults of test \"%s\"\n", test_name);
   FPRINTF (stderr, "Test time %s\n",
-           GNUNET_STRINGS_relative_time_to_string (total_time,
-                                                   GNUNET_YES));
-  FPRINTF (stderr, "Test bandwidth: %f kb/s\n",
-           4 * TOTAL_PACKETS * 1.0 / (total_time.rel_value_us / 1000)); // 4bytes * ms
-  FPRINTF (stderr, "Test throughput: %f packets/s\n\n",
-           TOTAL_PACKETS * 1000.0 / (total_time.rel_value_us / 1000)); // packets * ms
+           GNUNET_STRINGS_relative_time_to_string (total_time, GNUNET_YES));
+  FPRINTF (stderr, "Test bandwidth: %f kb/s\n", 4 * total_packets * 1.0 / (total_time.rel_value_us / 1000));    // 4bytes * ms
+  FPRINTF (stderr, "Test throughput: %f packets/s\n\n", total_packets * 1000.0 / (total_time.rel_value_us / 1000));     // packets * ms
   GAUGER ("CADET", test_name,
-          TOTAL_PACKETS * 1000.0 / (total_time.rel_value_us / 1000),
+          total_packets * 1000.0 / (total_time.rel_value_us / 1000),
           "packets/s");
 }
 
@@ -281,29 +294,19 @@ disconnect_cadet_peers (void *cls)
 
   disconnect_task = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "disconnecting cadet service of peers, called from line %ld\n",
-              line);
+              "disconnecting cadet service of peers, called from line %ld\n",
+              line);
   for (i = 0; i < 2; i++)
   {
     GNUNET_TESTBED_operation_done (t_op[i]);
   }
-  if (NULL != ch)
+  if (NULL != outgoing_ch)
   {
-    if (NULL != th)
-    {
-      GNUNET_CADET_notify_transmit_ready_cancel (th);
-      th = NULL;
-    }
-    GNUNET_CADET_channel_destroy (ch);
-    ch = NULL;
+    GNUNET_CADET_channel_destroy (outgoing_ch);
+    outgoing_ch = NULL;
   }
   if (NULL != incoming_ch)
   {
-    if (NULL != incoming_th)
-    {
-      GNUNET_CADET_notify_transmit_ready_cancel (incoming_th);
-      incoming_th = NULL;
-    }
     GNUNET_CADET_channel_destroy (incoming_ch);
     incoming_ch = NULL;
   }
@@ -322,10 +325,10 @@ static void
 shutdown_task (void *cls)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Ending test.\n");
-  if (NULL != data_job)
+  if (NULL != send_next_msg_task)
   {
-    GNUNET_SCHEDULER_cancel (data_job);
-    data_job = NULL;
+    GNUNET_SCHEDULER_cancel (send_next_msg_task);
+    send_next_msg_task = NULL;
   }
   if (NULL != test_task)
   {
@@ -335,8 +338,8 @@ shutdown_task (void *cls)
   if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
-    disconnect_task = GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers,
-                                                (void *) __LINE__);
+    disconnect_task =
+        GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers, (void *) __LINE__);
   }
 }
 
@@ -351,17 +354,11 @@ shutdown_task (void *cls)
  *          operation has executed successfully.
  */
 static void
-stats_cont (void *cls,
-            struct GNUNET_TESTBED_Operation *op,
-            const char *emsg)
+stats_cont (void *cls, struct GNUNET_TESTBED_Operation *op, const char *emsg)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              " KA sent: %u, KA received: %u\n",
-              ka_sent,
-              ka_received);
-  if ( (KEEPALIVE == test) &&
-       ( (ka_sent < 2) ||
-         (ka_sent > ka_received + 1)) )
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, " KA sent: %u, KA received: %u\n",
+              ka_sent, ka_received);
+  if ((KEEPALIVE == test) && ((ka_sent < 2) || (ka_sent > ka_received + 1)))
   {
     GNUNET_break (0);
     ok--;
@@ -370,8 +367,7 @@ stats_cont (void *cls,
 
   if (NULL != disconnect_task)
     GNUNET_SCHEDULER_cancel (disconnect_task);
-  disconnect_task = GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers,
-                                              cls);
+  disconnect_task = GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers, cls);
 }
 
 
@@ -387,11 +383,8 @@ stats_cont (void *cls,
  * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
  */
 static int
-stats_iterator (void *cls,
-                const struct GNUNET_TESTBED_Peer *peer,
-                const char *subsystem,
-                const char *name,
-                uint64_t value,
+stats_iterator (void *cls, const struct GNUNET_TESTBED_Peer *peer,
+                const char *subsystem, const char *name, uint64_t value,
                 int is_persistent)
 {
   static const char *s_sent = "# keepalives sent";
@@ -401,19 +394,15 @@ stats_iterator (void *cls,
   uint32_t i;
 
   i = GNUNET_TESTBED_get_index (peer);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "STATS PEER %u - %s [%s]: %llu\n",
-              i,
-              subsystem,
-              name,
-              (unsigned long long) value);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "STATS PEER %u - %s [%s]: %llu\n", i,
+              subsystem, name, (unsigned long long) value);
   if (0 == strncmp (s_sent, name, strlen (s_sent)) && 0 == i)
     ka_sent = value;
-  if (0 == strncmp(s_recv, name, strlen (s_recv)) && peers_requested - 1 == i)
+  if (0 == strncmp (s_recv, name, strlen (s_recv)) && peers_requested - 1 == i)
     ka_received = value;
-  if (0 == strncmp(rdrops, name, strlen (rdrops)))
+  if (0 == strncmp (rdrops, name, strlen (rdrops)))
     msg_dropped += value;
-  if (0 == strncmp(cdrops, name, strlen (cdrops)))
+  if (0 == strncmp (cdrops, name, strlen (cdrops)))
     msg_dropped += value;
 
   return GNUNET_OK;
@@ -423,7 +412,7 @@ stats_iterator (void *cls,
 /**
  * Task to gather all statistics.
  *
- * @param cls Closure (NULL).
+ * @param cls Closure (line from which the task was scheduled).
  */
 static void
 gather_stats_and_exit (void *cls)
@@ -432,21 +421,20 @@ gather_stats_and_exit (void *cls)
 
   disconnect_task = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "gathering statistics from line %d\n",
-              (int) l);
-  if (NULL != ch)
+              "gathering statistics from line %ld\n",
+              l);
+  if (NULL != outgoing_ch)
   {
-    if (NULL != th)
-    {
-      GNUNET_CADET_notify_transmit_ready_cancel (th);
-      th = NULL;
-    }
-    GNUNET_CADET_channel_destroy (ch);
-    ch = NULL;
+    GNUNET_CADET_channel_destroy (outgoing_ch);
+    outgoing_ch = NULL;
   }
-  stats_op = GNUNET_TESTBED_get_statistics (peers_running, testbed_peers,
-                                            "cadet", NULL,
-                                            &stats_iterator, stats_cont, cls);
+  stats_op = GNUNET_TESTBED_get_statistics (peers_running,
+                                            testbed_peers,
+                                            "cadet",
+                                            NULL,
+                                            &stats_iterator,
+                                            stats_cont,
+                                            cls);
 }
 
 
@@ -462,163 +450,151 @@ abort_test (long line)
   if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Aborting test from %ld\n", line);
-    disconnect_task = GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers,
-                                                (void *) line);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Aborting test from %ld\n", line);
+    disconnect_task =
+        GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers, (void *) line);
   }
 }
 
-/**
- * Transmit ready callback.
- *
- * @param cls Closure (message type).
- * @param size Size of the tranmist buffer.
- * @param buf Pointer to the beginning of the buffer.
- *
- * @return Number of bytes written to buf.
- */
-static size_t
-tmt_rdy (void *cls, size_t size, void *buf);
-
 
 /**
- * Task to request a new data transmission.
+ * Send a message on the channel with the appropriate size and payload.
+ *
+ * Update the appropriate *_sent counter.
  *
- * @param cls Closure (peer #).
+ * @param channel Channel to send the message on.
  */
 static void
-data_task (void *cls)
+send_test_message (struct GNUNET_CADET_Channel *channel)
 {
-  struct GNUNET_CADET_Channel *channel;
-  static struct GNUNET_CADET_TransmitHandle **pth;
-  long src;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_MessageHeader *msg;
+  uint32_t *data;
+  int payload;
+  int size;
 
-  data_job = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Data task\n");
-  if (GNUNET_YES == test_backwards)
-  {
-    channel = incoming_ch;
-    pth = &incoming_th;
-    src = peers_requested - 1;
-  }
-  else
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending test message on channel %p\n",
+              channel);
+  size = size_payload;
+  if (GNUNET_NO == initialized)
   {
-    channel = ch;
-    pth = &th;
-    src = 0;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending INITIALIZER\n");
+    size += 1000;
+    payload = data_sent;
+    if (SPEED_ACK == test) // FIXME unify SPEED_ACK with an initializer
+        data_sent++;
   }
-
-  GNUNET_assert (NULL != channel);
-  GNUNET_assert (NULL == *pth);
-
-  *pth = GNUNET_CADET_notify_transmit_ready (channel, GNUNET_NO,
-                                             GNUNET_TIME_UNIT_FOREVER_REL,
-                                             size_payload + data_sent,
-                                             &tmt_rdy, (void *) src);
-  if (NULL == *pth)
+  else if (SPEED == test || SPEED_ACK == test)
   {
-    unsigned long i = (unsigned long) cls;
-
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Retransmission\n");
-    if (0 == i)
+    if (get_target_channel() == channel)
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "  in 1 ms\n");
-      data_job = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS,
-                                               &data_task, (void *) 1L);
+      payload = ack_sent;
+      size += ack_sent;
+      ack_sent++;
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Sending ACK %u [%d bytes]\n",
+                  payload, size);
     }
     else
     {
-      i++;
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                  "in %llu ms\n",
-                  (unsigned long long) i);
-      data_job = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
-                                                                              i),
-                                               &data_task, (void *) i);
+      payload = data_sent;
+      size += data_sent;
+      data_sent++;
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Sending DATA %u [%d bytes]\n",
+                  data_sent, size);
     }
   }
-}
+  else if (FORWARD == test)
+  {
+    payload = ack_sent;
+  }
+  else if (P2P_SIGNAL == test)
+  {
+    payload = data_sent;
+  }
+  else
+  {
+    GNUNET_assert (0);
+  }
+  env = GNUNET_MQ_msg_extra (msg, size, GNUNET_MESSAGE_TYPE_DUMMY);
 
+  data = (uint32_t *) &msg[1];
+  *data = htonl (payload);
+  GNUNET_MQ_send (GNUNET_CADET_get_mq (channel), env);
+}
 
 /**
- * Transmit ready callback
+ * Task to request a new data transmission in a SPEED test, without waiting
+ * for previous messages to be sent/arrrive.
  *
- * @param cls Closure (peer # which is sending the data).
- * @param size Size of the buffer we have.
- * @param buf Buffer to copy data to.
+ * @param cls Closure (unused).
  */
-static size_t
-tmt_rdy (void *cls, size_t size, void *buf)
+static void
+send_next_msg (void *cls)
 {
-  struct GNUNET_MessageHeader *msg = buf;
-  size_t msg_size;
-  uint32_t *data;
-  long id = (long) cls;
-  unsigned int counter;
+  struct GNUNET_CADET_Channel *channel;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "tmt_rdy on %ld, filling buffer\n",
-              id);
-  if (0 == id)
-    th = NULL;
-  else if ((peers_requested - 1) == id)
-    incoming_th = NULL;
-  else
-    GNUNET_assert (0);
-  counter = get_expected_target () == id ? ack_sent : data_sent;
-  msg_size = size_payload + counter;
-  GNUNET_assert (msg_size > sizeof (struct GNUNET_MessageHeader));
-  if ( (size < msg_size) ||
-       (NULL == buf) )
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "size %u, buf %p, data_sent %u, ack_received %u\n",
-                (unsigned int) size,
-                buf,
-                data_sent,
-                ack_received);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ok %u, ok goal %u\n", ok, ok_goal);
-    GNUNET_break (ok >= ok_goal - 2);
-
-    return 0;
-  }
-  msg->size = htons (msg_size);
-  msg->type = htons (GNUNET_MESSAGE_TYPE_DUMMY);
-  data = (uint32_t *) &msg[1];
-  *data = htonl (counter);
-  if (GNUNET_NO == initialized)
+  send_next_msg_task = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending next message: %d\n", data_sent);
+
+  channel = GNUNET_YES == test_backwards ? incoming_ch : outgoing_ch;
+  GNUNET_assert (NULL != channel);
+  GNUNET_assert (SPEED == test);
+  send_test_message (channel);
+  if (data_sent < total_packets)
   {
+    /* SPEED test: Send all messages as soon as possible */
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "sending initializer\n");
-    msg_size = size_payload + 1000;
-    msg->size = htons (msg_size);
-    if (SPEED_ACK == test)
-        data_sent++;
+                "Scheduling message %d\n",
+                data_sent + 1);
+    send_next_msg_task =
+        GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_UNIT_SECONDS,
+                                     &send_next_msg,
+                                     NULL);
   }
-  else if ( (SPEED == test) ||
-            (SPEED_ACK == test) )
+}
+
+
+/**
+ * Every few messages cancel the timeout task and re-schedule it again, to
+ * avoid timing out when traffic keeps coming.
+ *
+ * @param line Code line number to log if a timeout occurs.
+ */
+static void
+reschedule_timeout_task (long line)
+{
+  if ((ok % 10) == 0)
   {
-    if (get_expected_target() == id)
-      ack_sent++;
-    else
-      data_sent++;
-    counter++;
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                " Sent message %u size %u\n",
-                counter,
-                (unsigned int) msg_size);
-    if ( (data_sent < TOTAL_PACKETS) &&
-         (SPEED == test) )
+    if (NULL != disconnect_task)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  " Scheduling message %d\n",
-                  counter + 1);
-      data_job = GNUNET_SCHEDULER_add_now (&data_task, NULL);
+                  " reschedule timeout every 10 messages\n");
+      GNUNET_SCHEDULER_cancel (disconnect_task);
+      disconnect_task = GNUNET_SCHEDULER_add_delayed (short_time,
+                                                      &gather_stats_and_exit,
+                                                      (void *) line);
     }
   }
+}
+
 
-  return msg_size;
+/**
+ * Check if payload is sane (size contains payload).
+ *
+ * @param cls should match #ch
+ * @param message The actual message.
+ * @return #GNUNET_OK to keep the channel open,
+ *         #GNUNET_SYSERR to close it (signal serious error).
+ */
+static int
+check_data (void *cls, const struct GNUNET_MessageHeader *message)
+{
+  if (sizeof (struct GNUNET_MessageHeader) >= ntohs (message->size))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;             /* all is well-formed */
 }
 
 
@@ -626,75 +602,50 @@ tmt_rdy (void *cls, size_t size, void *buf)
  * Function is called whenever a message is received.
  *
  * @param cls closure (set from GNUNET_CADET_connect(), peer number)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the channel
  * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
  */
-static int
-data_callback (void *cls,
-               struct GNUNET_CADET_Channel *channel,
-               void **channel_ctx,
-               const struct GNUNET_MessageHeader *message)
+static void
+handle_data (void *cls, const struct GNUNET_MessageHeader *message)
 {
-  struct GNUNET_CADET_TransmitHandle **pth;
-  long client = (long) cls;
-  long expected_target_client;
+  struct CadetTestChannelWrapper *ch = cls;
+  struct GNUNET_CADET_Channel *channel = ch->ch;
   uint32_t *data;
   uint32_t payload;
-  unsigned int counter;
+  int *counter;
 
   ok++;
-  counter = get_expected_target () == client ? data_received : ack_received;
-
   GNUNET_CADET_receive_done (channel);
+  counter = get_target_channel () == channel ? &data_received : &ack_received;
 
-  if ((ok % 10) == 0)
+  reschedule_timeout_task ((long) __LINE__);
+
+  if (channel == outgoing_ch)
   {
-    if (NULL != disconnect_task)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  " reschedule timeout\n");
-      GNUNET_SCHEDULER_cancel (disconnect_task);
-      disconnect_task = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
-                                                      &gather_stats_and_exit,
-                                                      (void *) __LINE__);
-    }
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Root client got a message.\n");
   }
-
-  switch (client)
+  else if (channel == incoming_ch)
   {
-  case 0L:
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Root client got a message!\n");
-    GNUNET_assert (channel == ch);
-    pth = &th;
-    break;
-  case 1L:
-  case 4L:
-    GNUNET_assert (client == peers_requested - 1);
-    GNUNET_assert (channel == incoming_ch);
-    pth = &incoming_th;
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Leaf client %ld got a message.\n",
-                client);
-    break;
-  default:
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Client %ld not valid.\n", client);
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Leaf client got a message.\n");
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Unknown channel %p.\n", channel);
     GNUNET_assert (0);
   }
+
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, " ok: (%d/%d)\n", ok, ok_goal);
   data = (uint32_t *) &message[1];
   payload = ntohl (*data);
-  if (payload == counter)
+  if (payload == *counter)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_INFO, " payload as expected: %u\n", payload);
   }
   else
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, " payload %u, expected: %u\n",
-                payload, counter);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                " payload %u, expected: %u\n",
+                payload, *counter);
   }
-  expected_target_client = get_expected_target ();
 
   if (GNUNET_NO == initialized)
   {
@@ -702,189 +653,152 @@ data_callback (void *cls,
     start_time = GNUNET_TIME_absolute_get ();
     if (SPEED == test)
     {
-      GNUNET_assert (peers_requested - 1 == client);
-      data_job = GNUNET_SCHEDULER_add_now (&data_task, NULL);
-      return GNUNET_OK;
+      GNUNET_assert (incoming_ch == channel);
+      send_next_msg_task = GNUNET_SCHEDULER_add_now (&send_next_msg, NULL);
+      return;
     }
   }
 
-  counter++;
-  if (client == expected_target_client) /* Normally 4 */
+  (*counter)++;
+  if (get_target_channel () == channel) /* Got "data" */
   {
-    data_received++;
     GNUNET_log (GNUNET_ERROR_TYPE_INFO, " received data %u\n", data_received);
     if (SPEED != test || (ok_goal - 2) == ok)
     {
       /* Send ACK */
-      GNUNET_assert (NULL == *pth);
-      *pth = GNUNET_CADET_notify_transmit_ready (channel, GNUNET_NO,
-                                                 GNUNET_TIME_UNIT_FOREVER_REL,
-                                                 size_payload + ack_sent,
-                                                 &tmt_rdy, (void *) client);
-      return GNUNET_OK;
+      send_test_message (channel);
+      return;
     }
     else
     {
-      if (data_received < TOTAL_PACKETS)
-        return GNUNET_OK;
+      if (data_received < total_packets)
+        return;
     }
   }
-  else /* Normally 0 */
+  else /* Got "ack" */
   {
     if (SPEED_ACK == test || SPEED == test)
     {
-      ack_received++;
       GNUNET_log (GNUNET_ERROR_TYPE_INFO, " received ack %u\n", ack_received);
-      /* send more data */
-      GNUNET_assert (NULL == *pth);
-      *pth = GNUNET_CADET_notify_transmit_ready (channel, GNUNET_NO,
-                                                 GNUNET_TIME_UNIT_FOREVER_REL,
-                                                 size_payload + data_sent,
-                                                 &tmt_rdy, (void *) client);
-      if (ack_received < TOTAL_PACKETS && SPEED != test)
-        return GNUNET_OK;
+      /* Send more data */
+      send_test_message (channel);
+      if (ack_received < total_packets && SPEED != test)
+        return;
       if (ok == 2 && SPEED == test)
-        return GNUNET_OK;
-      show_end_data();
+        return;
+      show_end_data ();
     }
     if (test == P2P_SIGNAL)
     {
-      if (NULL != incoming_th)
-      {
-        GNUNET_CADET_notify_transmit_ready_cancel (incoming_th);
-        incoming_th = NULL;
-      }
       GNUNET_CADET_channel_destroy (incoming_ch);
       incoming_ch = NULL;
     }
     else
     {
-      if (NULL != th)
-      {
-        GNUNET_CADET_notify_transmit_ready_cancel (th);
-        th = NULL;
-      }
-      GNUNET_CADET_channel_destroy (ch);
-      ch = NULL;
+      GNUNET_CADET_channel_destroy (outgoing_ch);
+      outgoing_ch = NULL;
     }
   }
-
-  return GNUNET_OK;
 }
 
 
 /**
- * Data handlers for every message type of CADET's payload.
- * {callback_function, message_type, size_expected}
- */
-static struct GNUNET_CADET_MessageHandler handlers[] = {
-  {&data_callback,
-   GNUNET_MESSAGE_TYPE_DUMMY,
-   sizeof (struct GNUNET_MessageHeader)},
-  {NULL, 0, 0}
-};
-
-
-/**
- * Method called whenever another peer has added us to a channel
- * the other peer initiated.
+ * Method called whenever a peer connects to a port in MQ-based CADET.
  *
- * @param cls Closure.
+ * @param cls Closure from #GNUNET_CADET_open_port (peer # as long).
  * @param channel New handle to the channel.
- * @param initiator Peer that started the channel.
- * @param port Port this channel is connected to.
- * @param options channel option flags
- * @return Initial channel context for the channel
- *         (can be NULL -- that's not an error).
+ * @param source Peer that started this channel.
+ * @return Closure for the incoming @a channel. It's given to:
+ *         - The #GNUNET_CADET_DisconnectEventHandler (given to
+ *           #GNUNET_CADET_open_port) when the channel dies.
+ *         - Each the #GNUNET_MQ_MessageCallback handlers for each message
+ *           received on the @a channel.
  */
 static void *
-incoming_channel (void *cls,
-                  struct GNUNET_CADET_Channel *channel,
-                  const struct GNUNET_PeerIdentity *initiator,
-                  const struct GNUNET_HashCode *port,
-                  enum GNUNET_CADET_ChannelOption options)
+connect_handler (void *cls, struct GNUNET_CADET_Channel *channel,
+                 const struct GNUNET_PeerIdentity *source)
 {
+  struct CadetTestChannelWrapper *ch;
+  long peer = (long) cls;
+
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Incoming channel from %s to peer %d:%s\n",
-              GNUNET_i2s (initiator),
-              (int) (long) cls, GNUNET_h2s (port));
+              "Incoming channel from %s to %ld: %p\n",
+              GNUNET_i2s (source), peer, channel);
   ok++;
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, " ok: %d\n", ok);
-  if ((long) cls == peers_requested - 1)
+  if (peer == peers_requested - 1)
   {
     if (NULL != incoming_ch)
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                  "Duplicate incoming channel for client %lu\n",
-                  (long) cls);
-      GNUNET_break(0);
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Duplicate incoming channel for client %lu\n", (long) cls);
+      GNUNET_assert (0);
     }
     incoming_ch = channel;
   }
   else
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "Incoming channel for unexpected peer #%lu\n",
-                (long) cls);
-    GNUNET_break (0);
+                "Incoming channel for unexpected peer #%lu\n", (long) cls);
+    GNUNET_assert (0);
   }
   if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
-    disconnect_task = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
+    disconnect_task = GNUNET_SCHEDULER_add_delayed (short_time,
                                                     &gather_stats_and_exit,
                                                     (void *) __LINE__);
   }
 
-  return NULL;
+  /* TODO: cannot return channel as-is, in order to unify the data handlers */
+  ch = GNUNET_new (struct CadetTestChannelWrapper);
+  ch->ch = channel;
+
+  return ch;
 }
 
 
 /**
- * Function called whenever an inbound channel is destroyed.  Should clean up
- * any associated state.
+ * Function called whenever an MQ-channel is destroyed, even if the destruction
+ * was requested by #GNUNET_CADET_channel_destroy.
+ * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
+ *
+ * It should clean up any associated state, including cancelling any pending
+ * transmission on this channel.
  *
- * @param cls closure (set from GNUNET_CADET_connect, peer number)
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
+ * @param cls Channel closure (channel wrapper).
+ * @param channel Connection to the other end (henceforth invalid).
  */
 static void
-channel_cleaner (void *cls,
-                 const struct GNUNET_CADET_Channel *channel,
-                 void *channel_ctx)
+disconnect_handler (void *cls, const struct GNUNET_CADET_Channel *channel)
 {
-  long i = (long) cls;
+  struct CadetTestChannelWrapper *ch_w = cls;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Incoming channel disconnected at peer %ld\n",
-              i);
-  if (peers_running - 1 == i)
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Channel disconnected\n");
+  GNUNET_assert (ch_w->ch == channel);
+  if (channel == incoming_ch)
   {
     ok++;
-    GNUNET_break (channel == incoming_ch);
     incoming_ch = NULL;
   }
-  else if (0L == i)
+  else if (outgoing_ch == channel
+  )
   {
     if (P2P_SIGNAL == test)
     {
       ok++;
     }
-    GNUNET_break (channel == ch);
-    ch = NULL;
+    outgoing_ch = NULL;
   }
   else
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "Unknown peer! %d\n",
-                (int) i);
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unknown channel! %p\n", channel);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, " ok: %d\n", ok);
 
   if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
-    disconnect_task = GNUNET_SCHEDULER_add_now (&gather_stats_and_exit,
-                                                (void *) __LINE__);
+    disconnect_task =
+        GNUNET_SCHEDULER_add_now (&gather_stats_and_exit, (void *) __LINE__);
   }
 }
 
@@ -898,13 +812,20 @@ channel_cleaner (void *cls,
  * @param cls Closure (unused).
  */
 static void
-do_test (void *cls)
+start_test (void *cls)
 {
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_var_size (data,
+                           GNUNET_MESSAGE_TYPE_DUMMY,
+                           struct GNUNET_MessageHeader,
+                           NULL),
+    GNUNET_MQ_handler_end ()
+  };
+  struct CadetTestChannelWrapper *ch;
   enum GNUNET_CADET_ChannelOption flags;
 
   test_task = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "do_test\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "start_test\n");
   if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
@@ -918,30 +839,33 @@ do_test (void *cls)
     flags |= GNUNET_CADET_OPTION_RELIABLE;
   }
 
-  ch = GNUNET_CADET_channel_create (h1,
-                                    NULL,
-                                    p_id[1],
-                                    &port,
-                                    flags);
+  ch = GNUNET_new (struct CadetTestChannelWrapper);
+  outgoing_ch = GNUNET_CADET_channel_create (h1,
+                                             ch,
+                                             p_id[1],
+                                             &port,
+                                             flags,
+                                             NULL,
+                                             &disconnect_handler,
+                                             handlers);
+
+  ch->ch = outgoing_ch;
 
-  disconnect_task
-    = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
-                                    &gather_stats_and_exit,
-                                    (void *) __LINE__);
+  disconnect_task = GNUNET_SCHEDULER_add_delayed (short_time,
+                                                  &gather_stats_and_exit,
+                                                  (void *) __LINE__);
   if (KEEPALIVE == test)
-    return; /* Don't send any data. */
+    return;                     /* Don't send any data. */
+
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending data initializer...\n");
   data_received = 0;
   data_sent = 0;
   ack_received = 0;
   ack_sent = 0;
-  th = GNUNET_CADET_notify_transmit_ready (ch,
-                                           GNUNET_NO,
-                                           GNUNET_TIME_UNIT_FOREVER_REL,
-                                           size_payload + 1000,
-                                           &tmt_rdy, (void *) 0L);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending data initializer on channel %p...\n",
+              outgoing_ch);
+  send_test_message (outgoing_ch);
 }
 
 
@@ -955,35 +879,26 @@ do_test (void *cls)
  *             NULL if the operation is successfull
  */
 static void
-pi_cb (void *cls,
-       struct GNUNET_TESTBED_Operation *op,
-       const struct GNUNET_TESTBED_PeerInformation *pinfo,
-       const char *emsg)
+pi_cb (void *cls, struct GNUNET_TESTBED_Operation *op,
+       const struct GNUNET_TESTBED_PeerInformation *pinfo, const char *emsg)
 {
   long i = (long) cls;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "id callback for %ld\n", i);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ID callback for %ld\n", i);
 
-  if ( (NULL == pinfo) ||
-       (NULL != emsg) )
+  if ((NULL == pinfo) || (NULL != emsg))
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "pi_cb: %s\n", emsg);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "pi_cb: %s\n", emsg);
     abort_test (__LINE__);
     return;
   }
   p_id[i] = pinfo->result.id;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "  id: %s\n", GNUNET_i2s (p_id[i]));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  id: %s\n", GNUNET_i2s (p_id[i]));
   p_ids++;
   if (p_ids < 2)
     return;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got all IDs, starting test\n");
-  test_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
-                                            &do_test,
-                                            NULL);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got all IDs, starting test\n");
+  test_task = GNUNET_SCHEDULER_add_now (&start_test, NULL);
 }
 
 
@@ -994,7 +909,7 @@ pi_cb (void *cls,
  * @param ctx Argument to give to GNUNET_CADET_TEST_cleanup on test end.
  * @param num_peers Number of peers that are running.
  * @param peers Array of peers.
- * @param cadetes Handle to each of the CADETs of the peers.
+ * @param cadets Handle to each of the CADETs of the peers.
  */
 static void
 tmain (void *cls,
@@ -1011,16 +926,18 @@ tmain (void *cls,
   testbed_peers = peers;
   h1 = cadets[0];
   h2 = cadets[num_peers - 1];
-  disconnect_task = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
+  disconnect_task = GNUNET_SCHEDULER_add_delayed (short_time,
                                                   &disconnect_cadet_peers,
                                                   (void *) __LINE__);
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
   t_op[0] = GNUNET_TESTBED_peer_get_information (peers[0],
                                                  GNUNET_TESTBED_PIT_IDENTITY,
-                                                 &pi_cb, (void *) 0L);
+                                                 &pi_cb,
+                                                 (void *) 0L);
   t_op[1] = GNUNET_TESTBED_peer_get_information (peers[num_peers - 1],
                                                  GNUNET_TESTBED_PIT_IDENTITY,
-                                                 &pi_cb, (void *) 1L);
+                                                 &pi_cb,
+                                                 (void *) 1L);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "requested peer ids\n");
 }
 
@@ -1031,16 +948,46 @@ tmain (void *cls,
 int
 main (int argc, char *argv[])
 {
-  initialized = GNUNET_NO;
   static const struct GNUNET_HashCode *ports[2];
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_var_size (data,
+                           GNUNET_MESSAGE_TYPE_DUMMY,
+                           struct GNUNET_MessageHeader,
+                           NULL),
+    GNUNET_MQ_handler_end ()
+  };
   const char *config_file;
   char port_id[] = "test port";
-  GNUNET_CRYPTO_hash (port_id, sizeof (port_id), &port);
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('t',
+                                            "time",
+                                            "short_time",
+                                            gettext_noop ("set short timeout"),
+                                            &short_time),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('m',
+                                   "messages",
+                                   "NUM_MESSAGES",
+                                   gettext_noop ("set number of messages to send"),
+                                   &total_packets),
+
+    GNUNET_GETOPT_OPTION_END
+  };
 
+
+  initialized = GNUNET_NO;
   GNUNET_log_setup ("test", "DEBUG", NULL);
-  config_file = "test_cadet.conf";
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Start\n");
+  total_packets = TOTAL_PACKETS;
+  short_time = SHORT_TIME;
+  if (-1 == GNUNET_GETOPT_run (argv[0], options, argc, argv))
+  {
+    FPRINTF (stderr, "test failed: problem with CLI parameters\n");
+    exit (1);
+  }
+
+  config_file = "test_cadet.conf";
+  GNUNET_CRYPTO_hash (port_id, sizeof (port_id), &port);
 
   /* Find out requested size */
   if (strstr (argv[0], "_2_") != NULL)
@@ -1078,11 +1025,11 @@ main (int argc, char *argv[])
   {
     /* Test is supposed to generate the following callbacks:
      * 1 incoming channel (@dest)
-     * TOTAL_PACKETS received data packet (@dest)
-     * TOTAL_PACKETS received data packet (@orig)
+     * total_packets received data packet (@dest)
+     * total_packets received data packet (@orig)
      * 1 received channel destroy (@dest)
      */
-    ok_goal = TOTAL_PACKETS * 2 + 2;
+    ok_goal = total_packets * 2 + 2;
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "SPEED_ACK\n");
     test = SPEED_ACK;
     test_name = "speed ack";
@@ -1092,11 +1039,11 @@ main (int argc, char *argv[])
     /* Test is supposed to generate the following callbacks:
      * 1 incoming channel (@dest)
      * 1 initial packet (@dest)
-     * TOTAL_PACKETS received data packet (@dest)
+     * total_packets received data packet (@dest)
      * 1 received data packet (@orig)
      * 1 received channel destroy (@dest)
      */
-    ok_goal = TOTAL_PACKETS + 4;
+    ok_goal = total_packets + 4;
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "SPEED\n");
     if (strstr (argv[0], "_reliable") != NULL)
     {
@@ -1137,22 +1084,22 @@ main (int argc, char *argv[])
   p_ids = 0;
   ports[0] = &port;
   ports[1] = NULL;
-  GNUNET_CADET_TEST_run ("test_cadet_small",
-                        config_file,
-                        peers_requested,
-                        &tmain,
-                        NULL, /* tmain cls */
-                        &incoming_channel,
-                        &channel_cleaner,
-                        handlers,
-                        ports);
+  GNUNET_CADET_TEST_ruN ("test_cadet_small",
+                         config_file,
+                         peers_requested,
+                         &tmain,
+                         NULL,        /* tmain cls */
+                         &connect_handler,
+                         NULL,
+                         &disconnect_handler,
+                         handlers,
+                         ports);
   if (NULL != strstr (argv[0], "_reliable"))
-    msg_dropped = 0; /* dropped should be retransmitted */
+    msg_dropped = 0;            /* dropped should be retransmitted */
 
   if (ok_goal > ok - msg_dropped)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "FAILED! (%d/%d)\n", ok, ok_goal);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "FAILED! (%d/%d)\n", ok, ok_goal);
     return 1;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "success\n");
diff --git a/src/cadet/test_cadet_local.c b/src/cadet/test_cadet_local.c
deleted file mode 100644
index 2b915ab81..000000000
--- a/src/cadet/test_cadet_local.c
+++ /dev/null
@@ -1,351 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/test_cadet_local.c
- * @brief test cadet local: test of cadet channels with just one peer
- * @author Bartlomiej Polot
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_testing_lib.h"
-#include "gnunet_cadet_service.h"
-
-struct GNUNET_TESTING_Peer *me;
-
-static struct GNUNET_CADET_Handle *cadet_peer_1;
-
-static struct GNUNET_CADET_Handle *cadet_peer_2;
-
-static struct GNUNET_CADET_Channel *ch;
-
-static int result = GNUNET_OK;
-
-static int got_data = GNUNET_NO;
-
-static struct GNUNET_SCHEDULER_Task *abort_task;
-
-static struct GNUNET_SCHEDULER_Task *connect_task;
-
-static struct GNUNET_CADET_TransmitHandle *mth;
-
-
-/**
- * Connect to other client and send data
- *
- * @param cls Closue (unused).
- */
-static void
-do_connect (void *cls);
-
-
-/**
- * Shutdown nicely
- */
-static void
-do_shutdown (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "shutdown\n");
-  if (NULL != abort_task)
-  {
-    GNUNET_SCHEDULER_cancel (abort_task);
-    abort_task = NULL;
-  }
-  if (NULL != ch)
-  {
-    GNUNET_CADET_channel_destroy (ch);
-    ch = NULL;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Disconnect client 1\n");
-  if (NULL != cadet_peer_1)
-  {
-    GNUNET_CADET_disconnect (cadet_peer_1);
-    cadet_peer_1 = NULL;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Disconnect client 2\n");
-  if (NULL != cadet_peer_2)
-  {
-    GNUNET_CADET_disconnect (cadet_peer_2);
-    cadet_peer_2 = NULL;
-  }
-  if (NULL != connect_task)
-  {
-    GNUNET_SCHEDULER_cancel (connect_task);
-    connect_task = NULL;
-  }
-}
-
-
-/**
- * Something went wrong and timed out. Kill everything and set error flag
- */
-static void
-do_abort (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ABORT\n");
-  result = GNUNET_SYSERR;
-  abort_task = NULL;
-  GNUNET_SCHEDULER_shutdown ();
-}
-
-
-/**
- * Function is called whenever a message is received.
- *
- * @param cls closure (set from GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the channel
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
- */
-static int
-data_callback (void *cls,
-               struct GNUNET_CADET_Channel *channel,
-               void **channel_ctx,
-               const struct GNUNET_MessageHeader *message)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Data callback! Shutting down.\n");
-  got_data = GNUNET_YES;
-  GNUNET_SCHEDULER_shutdown ();
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
-}
-
-
-/**
- * Method called whenever another peer has added us to a channel
- * the other peer initiated.
- *
- * @param cls closure
- * @param channel new handle to the channel
- * @param initiator peer that started the channel
- * @param port port number
- * @param options channel options
- * @return initial channel context for the channel
- *         (can be NULL -- that's not an error)
- */
-static void *
-inbound_channel (void *cls,
-                 struct GNUNET_CADET_Channel *channel,
-                 const struct GNUNET_PeerIdentity *initiator,
-                 const struct GNUNET_HashCode *port,
-                 enum GNUNET_CADET_ChannelOption options)
-{
-  long id = (long) cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "received incoming channel on peer %d, port %s\n",
-              (int) id,
-              GNUNET_h2s (port));
-  if (id != 2L)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "wrong peer\n");
-    result = GNUNET_SYSERR;
-  }
-  return NULL;
-}
-
-
-/**
- * Function called whenever an channel is destroyed.  Should clean up
- * any associated state.
- *
- * @param cls closure (set from GNUNET_CADET_connect)
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                    with the channel is stored
- */
-static void
-channel_end (void *cls,
-             const struct GNUNET_CADET_Channel *channel,
-             void *channel_ctx)
-{
-  long id = (long) cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "incoming channel closed at peer %ld\n",
-              id);
-  if (NULL != mth)
-  {
-    GNUNET_CADET_notify_transmit_ready_cancel (mth);
-    mth = NULL;
-  }
-  if (channel == ch)
-    ch = NULL;
-  if (GNUNET_NO == got_data)
-  {
-    if (NULL == connect_task)
-      connect_task
-        = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
-                                                                       2),
-                                        &do_connect,
-                                        NULL);
-  }
-}
-
-
-/**
- * Handler array for traffic received on peer1
- */
-static struct GNUNET_CADET_MessageHandler handlers1[] = {
-  {&data_callback, 1, 0},
-  {NULL, 0, 0}
-};
-
-
-/**
- * Handler array for traffic received on peer2 (none expected)
- */
-static struct GNUNET_CADET_MessageHandler handlers2[] = {
-  {&data_callback, 1, 0},
-  {NULL, 0, 0}
-};
-
-
-/**
- * Data send callback: fillbuffer with test packet.
- *
- * @param cls Closure (unused).
- * @param size Buffer size.
- * @param buf Buffer to fill.
- *
- * @return size of test packet.
- */
-static size_t
-do_send (void *cls, size_t size, void *buf)
-{
-  struct GNUNET_MessageHeader *m = buf;
-
-  mth = NULL;
-  if (NULL == buf)
-  {
-    GNUNET_break (0);
-    result = GNUNET_SYSERR;
-    return 0;
-  }
-  m->size = htons (sizeof (struct GNUNET_MessageHeader));
-  m->type = htons (1);
-  GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader));
-  return sizeof (struct GNUNET_MessageHeader);
-}
-
-
-/**
- * Connect to other client and send data
- *
- * @param cls Closue (unused).
- */
-static void
-do_connect (void *cls)
-{
-  struct GNUNET_PeerIdentity id;
-
-  connect_task = NULL;
-  GNUNET_TESTING_peer_get_identity (me, &id);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "CONNECT BY PORT\n");
-  ch = GNUNET_CADET_channel_create (cadet_peer_1,
-                                    NULL,
-                                    &id, GC_u2h (1),
-                                    GNUNET_CADET_OPTION_DEFAULT);
-  mth = GNUNET_CADET_notify_transmit_ready (ch, GNUNET_NO,
-                                            GNUNET_TIME_UNIT_FOREVER_REL,
-                                            sizeof (struct GNUNET_MessageHeader),
-                                            &do_send, NULL);
-}
-
-
-/**
- * Initialize framework and start test
- *
- * @param cls Closure (unused).
- * @param cfg Configuration handle.
- * @param peer Testing peer handle.
- */
-static void
-run (void *cls,
-     const struct GNUNET_CONFIGURATION_Handle *cfg,
-     struct GNUNET_TESTING_Peer *peer)
-{
-  me = peer;
-  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
-                                 NULL);
-  abort_task =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                    (GNUNET_TIME_UNIT_SECONDS, 15),
-                                    &do_abort,
-                                    NULL);
-  cadet_peer_1 = GNUNET_CADET_connect (cfg,       /* configuration */
-                                     (void *) 1L,       /* cls */
-                                     &channel_end,      /* channel end hndlr */
-                                     handlers1); /* traffic handlers */
-  cadet_peer_2 = GNUNET_CADET_connect (cfg,       /* configuration */
-                                     (void *) 2L,     /* cls */
-                                     &channel_end,      /* channel end hndlr */
-                                     handlers2); /* traffic handlers */
-
-  if ( (NULL == cadet_peer_1) ||
-       (NULL == cadet_peer_2) )
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Couldn't connect to cadet :(\n");
-    result = GNUNET_SYSERR;
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  GNUNET_CADET_open_port (cadet_peer_2,
-                          GC_u2h (1),
-                          &inbound_channel,
-                          (void *) 2L);
-  if (NULL == connect_task)
-    connect_task
-      = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
-                                                                     2),
-                                      &do_connect,
-                                      NULL);
-}
-
-
-/**
- * Main
- */
-int
-main (int argc, char *argv[])
-{
-  if (0 != GNUNET_TESTING_peer_run ("test-cadet-local",
-                                    "test_cadet.conf",
-                                &run, NULL))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "run failed\n");
-    return 2;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Final result: %d\n", result);
-  return (result == GNUNET_OK) ? 0 : 1;
-}
-
-/* end of test_cadet_local_1.c */
diff --git a/src/cadet/test_cadet_local_mq.c b/src/cadet/test_cadet_local_mq.c
index 19bafbed1..3089c7fbb 100644
--- a/src/cadet/test_cadet_local_mq.c
+++ b/src/cadet/test_cadet_local_mq.c
@@ -132,12 +132,12 @@ do_abort (void *cls)
 /**
  * Method called whenever a peer connects to a port in MQ-based CADET.
  *
- * @param cls Closure from #GNUNET_CADET_open_porT.
+ * @param cls Closure from #GNUNET_CADET_open_port.
  * @param channel New handle to the channel.
  * @param source Peer that started this channel.
  * @return Closure for the incoming @a channel. It's given to:
  *         - The #GNUNET_CADET_DisconnectEventHandler (given to
- *           #GNUNET_CADET_open_porT) when the channel dies.
+ *           #GNUNET_CADET_open_port) when the channel dies.
  *         - Each the #GNUNET_MQ_MessageCallback handlers for each message
  *           received on the @a channel.
  */
@@ -235,7 +235,7 @@ do_connect (void *cls)
   GNUNET_TESTING_peer_get_identity (me, &id);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "creating channel\n");
-  ch = GNUNET_CADET_channel_creatE (cadet_peer_1, /* cadet handle */
+  ch = GNUNET_CADET_channel_create (cadet_peer_1, /* cadet handle */
                                     NULL,         /* channel cls */
                                     &id,          /* destination */
                                     GC_u2h (TEST_MESSAGE_TYPE), /* port */
@@ -282,8 +282,8 @@ run (void *cls,
   abort_task = GNUNET_SCHEDULER_add_delayed (delay,
                                              &do_abort,
                                              (void *) (long) __LINE__);
-  cadet_peer_1 = GNUNET_CADET_connecT (cfg);
-  cadet_peer_2 = GNUNET_CADET_connecT (cfg);
+  cadet_peer_1 = GNUNET_CADET_connect (cfg);
+  cadet_peer_2 = GNUNET_CADET_connect (cfg);
 
   if ( (NULL == cadet_peer_1) ||
        (NULL == cadet_peer_2) )
@@ -297,7 +297,7 @@ run (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "CADET 1: %p\n", cadet_peer_1);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "CADET 2: %p\n", cadet_peer_2);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "handlers 2: %p\n", handlers);
-  GNUNET_CADET_open_porT (cadet_peer_2,          /* cadet handle */
+  GNUNET_CADET_open_port (cadet_peer_2,          /* cadet handle */
                           GC_u2h (TEST_PORT_ID), /* port id */
                           &connected,            /* connect handler */
                           (void *) 2L,           /* handle for #connected */
diff --git a/src/cadet/test_cadet_new.c b/src/cadet/test_cadet_new.c
deleted file mode 100644
index d32404815..000000000
--- a/src/cadet/test_cadet_new.c
+++ /dev/null
@@ -1,1048 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011, 2017 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file cadet/test_cadet_mq.c
- * @author Bart Polot
- * @author Christian Grothoff
- * @brief Test for the cadet service using mq API.
- */
-#include <stdio.h>
-#include "platform.h"
-#include "cadet_test_lib_new.h"
-#include "gnunet_cadet_service.h"
-#include "gnunet_statistics_service.h"
-#include <gauger.h>
-
-
-/**
- * Ugly workaround to unify data handlers on incoming and outgoing channels.
- */
-struct CadetTestChannelWrapper
-{
-  /**
-   * Channel pointer.
-   */
-  struct GNUNET_CADET_Channel *ch;
-};
-
-/**
- * How many messages to send
- */
-#define TOTAL_PACKETS 500       /* Cannot exceed 64k! */
-
-/**
- * How long until we give up on connecting the peers?
- */
-#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 120)
-
-/**
- * Time to wait for stuff that should be rather fast
- */
-#define SHORT_TIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 20)
-
-/**
- * DIFFERENT TESTS TO RUN
- */
-#define SETUP 0
-#define FORWARD 1
-#define KEEPALIVE 2
-#define SPEED 3
-#define SPEED_ACK 4
-#define SPEED_REL 8
-#define P2P_SIGNAL 10
-
-/**
- * Which test are we running?
- */
-static int test;
-
-/**
- * String with test name
- */
-static char *test_name;
-
-/**
- * Flag to send traffic leaf->root in speed tests to test BCK_ACK logic.
- */
-static int test_backwards = GNUNET_NO;
-
-/**
- * How many events have happened
- */
-static int ok;
-
-/**
- * Number of events expected to conclude the test successfully.
- */
-static int ok_goal;
-
-/**
- * Size of each test packet's payload
- */
-static size_t size_payload = sizeof (uint32_t);
-
-/**
- * Operation to get peer ids.
- */
-static struct GNUNET_TESTBED_Operation *t_op[2];
-
-/**
- * Peer ids.
- */
-static struct GNUNET_PeerIdentity *p_id[2];
-
-/**
- * Port ID
- */
-static struct GNUNET_HashCode port;
-
-/**
- * Peer ids counter.
- */
-static unsigned int p_ids;
-
-/**
- * Is the setup initialized?
- */
-static int initialized;
-
-/**
- * Number of payload packes sent.
- */
-static int data_sent;
-
-/**
- * Number of payload packets received.
- */
-static int data_received;
-
-/**
- * Number of payload packed acknowledgements sent.
- */
-static int ack_sent;
-
-/**
- * Number of payload packed explicitly (app level) acknowledged.
- */
-static int ack_received;
-
-/**
- * Total number of peers asked to run.
- */
-static unsigned long long peers_requested;
-
-/**
- * Number of currently running peers (should be same as @c peers_requested).
- */
-static unsigned long long peers_running;
-
-/**
- * Test context (to shut down).
- */
-struct GNUNET_CADET_TEST_Context *test_ctx;
-
-/**
- * Task called to disconnect peers.
- */
-static struct GNUNET_SCHEDULER_Task *disconnect_task;
-
-/**
- * Task To perform tests
- */
-static struct GNUNET_SCHEDULER_Task *test_task;
-
-/**
- * Task runnining #send_next_msg().
- */
-static struct GNUNET_SCHEDULER_Task *send_next_msg_task;
-
-/**
- * Cadet handle for the root peer
- */
-static struct GNUNET_CADET_Handle *h1;
-
-/**
- * Cadet handle for the first leaf peer
- */
-static struct GNUNET_CADET_Handle *h2;
-
-/**
- * Channel handle for the root peer
- */
-static struct GNUNET_CADET_Channel *outgoing_ch;
-
-/**
- * Channel handle for the dest peer
- */
-static struct GNUNET_CADET_Channel *incoming_ch;
-
-/**
- * Time we started the data transmission (after channel has been established
- * and initilized).
- */
-static struct GNUNET_TIME_Absolute start_time;
-
-/**
- * Peers handle.
- */
-static struct GNUNET_TESTBED_Peer **testbed_peers;
-
-/**
- * Statistics operation handle.
- */
-static struct GNUNET_TESTBED_Operation *stats_op;
-
-/**
- * Keepalives sent.
- */
-static unsigned int ka_sent;
-
-/**
- * Keepalives received.
- */
-static unsigned int ka_received;
-
-/**
- * How many messages were dropped by CADET because of full buffers?
- */
-static unsigned int msg_dropped;
-
-
-/******************************************************************************/
-
-
-/******************************************************************************/
-
-
-/**
- * Get the channel considered as the "target" or "receiver", depending on
- * the test type and size.
- *
- * @return Channel handle of the target client, either 0 (for backward tests)
- *         or the last peer in the line (for other tests).
- */
-static struct GNUNET_CADET_Channel *
-get_target_channel ()
-{
-  if (SPEED == test && GNUNET_YES == test_backwards)
-    return outgoing_ch;
-  else
-    return incoming_ch;
-}
-
-
-/**
- * Show the results of the test (banwidth acheived) and log them to GAUGER
- */
-static void
-show_end_data (void)
-{
-  static struct GNUNET_TIME_Absolute end_time;
-  static struct GNUNET_TIME_Relative total_time;
-
-  end_time = GNUNET_TIME_absolute_get ();
-  total_time = GNUNET_TIME_absolute_get_difference (start_time, end_time);
-  FPRINTF (stderr, "\nResults of test \"%s\"\n", test_name);
-  FPRINTF (stderr, "Test time %s\n",
-           GNUNET_STRINGS_relative_time_to_string (total_time, GNUNET_YES));
-  FPRINTF (stderr, "Test bandwidth: %f kb/s\n", 4 * TOTAL_PACKETS * 1.0 / (total_time.rel_value_us / 1000));    // 4bytes * ms
-  FPRINTF (stderr, "Test throughput: %f packets/s\n\n", TOTAL_PACKETS * 1000.0 / (total_time.rel_value_us / 1000));     // packets * ms
-  GAUGER ("CADET", test_name,
-          TOTAL_PACKETS * 1000.0 / (total_time.rel_value_us / 1000),
-          "packets/s");
-}
-
-
-/**
- * Disconnect from cadet services af all peers, call shutdown.
- *
- * @param cls Closure (line number from which termination was requested).
- * @param tc Task Context.
- */
-static void
-disconnect_cadet_peers (void *cls)
-{
-  long line = (long) cls;
-  unsigned int i;
-
-  disconnect_task = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "disconnecting cadet service of peers, called from line %ld\n",
-              line);
-  for (i = 0; i < 2; i++)
-  {
-    GNUNET_TESTBED_operation_done (t_op[i]);
-  }
-  if (NULL != outgoing_ch)
-  {
-    GNUNET_CADET_channel_destroy (outgoing_ch);
-    outgoing_ch = NULL;
-  }
-  if (NULL != incoming_ch)
-  {
-    GNUNET_CADET_channel_destroy (incoming_ch);
-    incoming_ch = NULL;
-  }
-  GNUNET_CADET_TEST_cleanup (test_ctx);
-  GNUNET_SCHEDULER_shutdown ();
-}
-
-
-/**
- * Shut down peergroup, clean up.
- *
- * @param cls Closure (unused).
- * @param tc Task Context.
- */
-static void
-shutdown_task (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Ending test.\n");
-  if (NULL != send_next_msg_task)
-  {
-    GNUNET_SCHEDULER_cancel (send_next_msg_task);
-    send_next_msg_task = NULL;
-  }
-  if (NULL != test_task)
-  {
-    GNUNET_SCHEDULER_cancel (test_task);
-    test_task = NULL;
-  }
-  if (NULL != disconnect_task)
-  {
-    GNUNET_SCHEDULER_cancel (disconnect_task);
-    disconnect_task =
-        GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers, (void *) __LINE__);
-  }
-}
-
-
-/**
- * Stats callback. Finish the stats testbed operation and when all stats have
- * been iterated, shutdown the test.
- *
- * @param cls Closure (line number from which termination was requested).
- * @param op the operation that has been finished
- * @param emsg error message in case the operation has failed; will be NULL if
- *          operation has executed successfully.
- */
-static void
-stats_cont (void *cls, struct GNUNET_TESTBED_Operation *op, const char *emsg)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, " KA sent: %u, KA received: %u\n",
-              ka_sent, ka_received);
-  if ((KEEPALIVE == test) && ((ka_sent < 2) || (ka_sent > ka_received + 1)))
-  {
-    GNUNET_break (0);
-    ok--;
-  }
-  GNUNET_TESTBED_operation_done (stats_op);
-
-  if (NULL != disconnect_task)
-    GNUNET_SCHEDULER_cancel (disconnect_task);
-  disconnect_task = GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers, cls);
-}
-
-
-/**
- * Process statistic values.
- *
- * @param cls closure (line number, unused)
- * @param peer the peer the statistic belong to
- * @param subsystem name of subsystem that created the statistic
- * @param name the name of the datum
- * @param value the current value
- * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not
- * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
- */
-static int
-stats_iterator (void *cls, const struct GNUNET_TESTBED_Peer *peer,
-                const char *subsystem, const char *name, uint64_t value,
-                int is_persistent)
-{
-  static const char *s_sent = "# keepalives sent";
-  static const char *s_recv = "# keepalives received";
-  static const char *rdrops = "# messages dropped due to full buffer";
-  static const char *cdrops = "# messages dropped due to slow client";
-  uint32_t i;
-
-  i = GNUNET_TESTBED_get_index (peer);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "STATS PEER %u - %s [%s]: %llu\n", i,
-              subsystem, name, (unsigned long long) value);
-  if (0 == strncmp (s_sent, name, strlen (s_sent)) && 0 == i)
-    ka_sent = value;
-  if (0 == strncmp (s_recv, name, strlen (s_recv)) && peers_requested - 1 == i)
-    ka_received = value;
-  if (0 == strncmp (rdrops, name, strlen (rdrops)))
-    msg_dropped += value;
-  if (0 == strncmp (cdrops, name, strlen (cdrops)))
-    msg_dropped += value;
-
-  return GNUNET_OK;
-}
-
-
-/**
- * Task to gather all statistics.
- *
- * @param cls Closure (line from which the task was scheduled).
- */
-static void
-gather_stats_and_exit (void *cls)
-{
-  long l = (long) cls;
-
-  disconnect_task = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "gathering statistics from line %ld\n",
-              l);
-  if (NULL != outgoing_ch)
-  {
-    GNUNET_CADET_channel_destroy (outgoing_ch);
-    outgoing_ch = NULL;
-  }
-  stats_op = GNUNET_TESTBED_get_statistics (peers_running,
-                                            testbed_peers,
-                                            "cadet",
-                                            NULL,
-                                            &stats_iterator,
-                                            stats_cont,
-                                            cls);
-}
-
-
-
-/**
- * Abort test: schedule disconnect and shutdown immediately
- *
- * @param line Line in the code the abort is requested from (__LINE__).
- */
-static void
-abort_test (long line)
-{
-  if (NULL != disconnect_task)
-  {
-    GNUNET_SCHEDULER_cancel (disconnect_task);
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Aborting test from %ld\n", line);
-    disconnect_task =
-        GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers, (void *) line);
-  }
-}
-
-
-/**
- * Send a message on the channel with the appropriate size and payload.
- *
- * Update the appropriate *_sent counter.
- *
- * @param channel Channel to send the message on.
- */
-static void
-send_test_message (struct GNUNET_CADET_Channel *channel)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *msg;
-  uint32_t *data;
-  int *counter;
-  int size;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending test message on channel %p\n",
-              channel);
-  size = size_payload;
-  if (GNUNET_NO == initialized)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending INITIALIZER\n");
-    size += 1000;
-    counter = &data_sent;
-    if (SPEED_ACK == test) // FIXME unify SPEED_ACK with an initializer
-        data_sent++;
-  }
-  else if (SPEED == test || SPEED_ACK == test)
-  {
-    counter = get_target_channel() == channel ? &ack_sent : &data_sent;
-    size += *counter;
-    *counter = *counter + 1;
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Sending message %u\n", *counter);
-  }
-  else
-  {
-    counter =  &ack_sent;
-  }
-  env = GNUNET_MQ_msg_extra (msg, size, GNUNET_MESSAGE_TYPE_DUMMY);
-
-  data = (uint32_t *) &msg[1];
-  *data = htonl (*counter);
-  GNUNET_MQ_send (GNUNET_CADET_get_mq (channel), env);
-}
-
-/**
- * Task to request a new data transmission in a SPEED test, without waiting
- * for previous messages to be sent/arrrive.
- *
- * @param cls Closure (unused).
- */
-static void
-send_next_msg (void *cls)
-{
-  struct GNUNET_CADET_Channel *channel;
-
-  send_next_msg_task = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending next message: %d\n", data_sent);
-
-  channel = GNUNET_YES == test_backwards ? incoming_ch : outgoing_ch;
-  GNUNET_assert (NULL != channel);
-  GNUNET_assert (SPEED == test);
-  send_test_message (channel);
-  if (data_sent < TOTAL_PACKETS)
-  {
-    /* SPEED test: Send all messages as soon as possible */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Scheduling message %d\n",
-                data_sent + 1);
-    send_next_msg_task = GNUNET_SCHEDULER_add_now (&send_next_msg, NULL);
-  }
-}
-
-
-/**
- * Every few messages cancel the timeout task and re-schedule it again, to
- * avoid timing out when traffic keeps coming.
- *
- * @param line Code line number to log if a timeout occurs.
- */
-static void
-reschedule_timeout_task (long line)
-{
-  if ((ok % 10) == 0)
-  {
-    if (NULL != disconnect_task)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  " reschedule timeout every 10 messages\n");
-      GNUNET_SCHEDULER_cancel (disconnect_task);
-      disconnect_task = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
-                                                      &gather_stats_and_exit,
-                                                      (void *) line);
-    }
-  }
-}
-
-
-/**
- * Check if payload is sane (size contains payload).
- *
- * @param cls should match #ch
- * @param message The actual message.
- * @return #GNUNET_OK to keep the channel open,
- *         #GNUNET_SYSERR to close it (signal serious error).
- */
-static int
-check_data (void *cls, const struct GNUNET_MessageHeader *message)
-{
-  if (sizeof (struct GNUNET_MessageHeader) >= ntohs (message->size))
-    return GNUNET_SYSERR;
-  return GNUNET_OK;             /* all is well-formed */
-}
-
-
-/**
- * Function is called whenever a message is received.
- *
- * @param cls closure (set from GNUNET_CADET_connect(), peer number)
- * @param message the actual message
- */
-static void
-handle_data (void *cls, const struct GNUNET_MessageHeader *message)
-{
-  struct CadetTestChannelWrapper *ch = cls;
-  struct GNUNET_CADET_Channel *channel = ch->ch;
-  uint32_t *data;
-  uint32_t payload;
-  int *counter;
-
-  ok++;
-  counter = get_target_channel () == channel ? &data_received : &ack_received;
-
-  reschedule_timeout_task ((long) __LINE__);
-
-  if (channel == outgoing_ch)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Root client got a message!\n");
-  }
-  else if (channel == incoming_ch)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Leaf client got a message.\n");
-  }
-  else
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Unknown channel %p.\n", channel);
-    GNUNET_assert (0);
-  }
-
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, " ok: (%d/%d)\n", ok, ok_goal);
-  data = (uint32_t *) &message[1];
-  payload = ntohl (*data);
-  if (payload == *counter)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, " payload as expected: %u\n", payload);
-  }
-  else
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                " payload %u, expected: %u\n",
-                payload, *counter);
-  }
-
-  if (GNUNET_NO == initialized)
-  {
-    initialized = GNUNET_YES;
-    start_time = GNUNET_TIME_absolute_get ();
-    if (SPEED == test)
-    {
-      GNUNET_assert (incoming_ch == channel);
-      send_next_msg_task = GNUNET_SCHEDULER_add_now (&send_next_msg, NULL);
-      return;
-    }
-  }
-
-  (*counter)++;
-  if (get_target_channel () == channel) /* Got "data" */
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, " received data %u\n", data_received);
-    if (SPEED != test || (ok_goal - 2) == ok)
-    {
-      /* Send ACK */
-      send_test_message (channel);
-      return;
-    }
-    else
-    {
-      if (data_received < TOTAL_PACKETS)
-        return;
-    }
-  }
-  else /* Got "ack" */
-  {
-    if (SPEED_ACK == test || SPEED == test)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO, " received ack %u\n", ack_received);
-      /* Send more data */
-      send_test_message (channel);
-      if (ack_received < TOTAL_PACKETS && SPEED != test)
-        return;
-      if (ok == 2 && SPEED == test)
-        return;
-      show_end_data ();
-    }
-    if (test == P2P_SIGNAL)
-    {
-      GNUNET_CADET_channel_destroy (incoming_ch);
-      incoming_ch = NULL;
-    }
-    else
-    {
-      GNUNET_CADET_channel_destroy (outgoing_ch);
-      outgoing_ch = NULL;
-    }
-  }
-}
-
-
-/**
- * Method called whenever a peer connects to a port in MQ-based CADET.
- *
- * @param cls Closure from #GNUNET_CADET_open_porT (peer # as long).
- * @param channel New handle to the channel.
- * @param source Peer that started this channel.
- * @return Closure for the incoming @a channel. It's given to:
- *         - The #GNUNET_CADET_DisconnectEventHandler (given to
- *           #GNUNET_CADET_open_porT) when the channel dies.
- *         - Each the #GNUNET_MQ_MessageCallback handlers for each message
- *           received on the @a channel.
- */
-static void *
-connect_handler (void *cls, struct GNUNET_CADET_Channel *channel,
-                 const struct GNUNET_PeerIdentity *source)
-{
-  struct CadetTestChannelWrapper *ch;
-  long peer = (long) cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Incoming channel from %s to peer %ld\n",
-              GNUNET_i2s (source), peer);
-  ok++;
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, " ok: %d\n", ok);
-  if (peer == peers_requested - 1)
-  {
-    if (NULL != incoming_ch)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  "Duplicate incoming channel for client %lu\n", (long) cls);
-      GNUNET_assert (0);
-    }
-    incoming_ch = channel;
-  }
-  else
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "Incoming channel for unexpected peer #%lu\n", (long) cls);
-    GNUNET_assert (0);
-  }
-  if (NULL != disconnect_task)
-  {
-    GNUNET_SCHEDULER_cancel (disconnect_task);
-    disconnect_task =
-        GNUNET_SCHEDULER_add_delayed (SHORT_TIME, &gather_stats_and_exit,
-                                      (void *) __LINE__);
-  }
-
-  /* TODO: cannot return channel as-is, in order to unify the data handlers */
-  ch = GNUNET_new (struct CadetTestChannelWrapper);
-  ch->ch = channel;
-
-  return ch;
-}
-
-
-/**
- * Function called whenever an MQ-channel is destroyed, even if the destruction
- * was requested by #GNUNET_CADET_channel_destroy.
- * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
- *
- * It should clean up any associated state, including cancelling any pending
- * transmission on this channel.
- *
- * @param cls Channel closure (channel wrapper).
- * @param channel Connection to the other end (henceforth invalid).
- */
-static void
-disconnect_handler (void *cls, const struct GNUNET_CADET_Channel *channel)
-{
-  struct CadetTestChannelWrapper *ch_w = cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Channel disconnected\n");
-  GNUNET_assert (ch_w->ch == channel);
-  if (channel == incoming_ch)
-  {
-    ok++;
-    incoming_ch = NULL;
-  }
-  else if (outgoing_ch == channel
-  )
-  {
-    if (P2P_SIGNAL == test)
-    {
-      ok++;
-    }
-    outgoing_ch = NULL;
-  }
-  else
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unknown channel! %p\n", channel);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, " ok: %d\n", ok);
-
-  if (NULL != disconnect_task)
-  {
-    GNUNET_SCHEDULER_cancel (disconnect_task);
-    disconnect_task =
-        GNUNET_SCHEDULER_add_now (&gather_stats_and_exit, (void *) __LINE__);
-  }
-}
-
-
-/**
- * START THE TESTCASE ITSELF, AS WE ARE CONNECTED TO THE CADET SERVICES.
- *
- * Testcase continues when the root receives confirmation of connected peers,
- * on callback function ch.
- *
- * @param cls Closure (unused).
- */
-static void
-start_test (void *cls)
-{
-  struct GNUNET_MQ_MessageHandler handlers[] = {
-    GNUNET_MQ_hd_var_size (data,
-                           GNUNET_MESSAGE_TYPE_DUMMY,
-                           struct GNUNET_MessageHeader,
-                           NULL),
-    GNUNET_MQ_handler_end ()
-  };
-  struct CadetTestChannelWrapper *ch;
-  enum GNUNET_CADET_ChannelOption flags;
-
-  test_task = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "start_test\n");
-  if (NULL != disconnect_task)
-  {
-    GNUNET_SCHEDULER_cancel (disconnect_task);
-    disconnect_task = NULL;
-  }
-
-  flags = GNUNET_CADET_OPTION_DEFAULT;
-  if (SPEED_REL == test)
-  {
-    test = SPEED;
-    flags |= GNUNET_CADET_OPTION_RELIABLE;
-  }
-
-  ch = GNUNET_new (struct CadetTestChannelWrapper);
-  outgoing_ch = GNUNET_CADET_channel_creatE (h1,
-                                             ch,
-                                             p_id[1],
-                                             &port,
-                                             flags,
-                                             NULL,
-                                             &disconnect_handler,
-                                             handlers);
-  ch->ch = outgoing_ch;
-
-  disconnect_task = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
-                                                  &gather_stats_and_exit,
-                                                  (void *) __LINE__);
-  if (KEEPALIVE == test)
-    return;                     /* Don't send any data. */
-
-
-  data_received = 0;
-  data_sent = 0;
-  ack_received = 0;
-  ack_sent = 0;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending data initializer...\n");
-  send_test_message (outgoing_ch);
-}
-
-
-/**
- * Callback to be called when the requested peer information is available
- *
- * @param cls the closure from GNUNET_TESTBED_peer_get_information()
- * @param op the operation this callback corresponds to
- * @param pinfo the result; will be NULL if the operation has failed
- * @param emsg error message if the operation has failed;
- *             NULL if the operation is successfull
- */
-static void
-pi_cb (void *cls, struct GNUNET_TESTBED_Operation *op,
-       const struct GNUNET_TESTBED_PeerInformation *pinfo, const char *emsg)
-{
-  long i = (long) cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ID callback for %ld\n", i);
-
-  if ((NULL == pinfo) || (NULL != emsg))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "pi_cb: %s\n", emsg);
-    abort_test (__LINE__);
-    return;
-  }
-  p_id[i] = pinfo->result.id;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  id: %s\n", GNUNET_i2s (p_id[i]));
-  p_ids++;
-  if (p_ids < 2)
-    return;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got all IDs, starting test\n");
-  test_task = GNUNET_SCHEDULER_add_now (&start_test, NULL);
-}
-
-
-/**
- * test main: start test when all peers are connected
- *
- * @param cls Closure.
- * @param ctx Argument to give to GNUNET_CADET_TEST_cleanup on test end.
- * @param num_peers Number of peers that are running.
- * @param peers Array of peers.
- * @param cadets Handle to each of the CADETs of the peers.
- */
-static void
-tmain (void *cls,
-       struct GNUNET_CADET_TEST_Context *ctx,
-       unsigned int num_peers,
-       struct GNUNET_TESTBED_Peer **peers,
-       struct GNUNET_CADET_Handle **cadets)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "test main\n");
-  ok = 0;
-  test_ctx = ctx;
-  peers_running = num_peers;
-  GNUNET_assert (peers_running == peers_requested);
-  testbed_peers = peers;
-  h1 = cadets[0];
-  h2 = cadets[num_peers - 1];
-  disconnect_task = GNUNET_SCHEDULER_add_delayed (SHORT_TIME,
-                                                  &disconnect_cadet_peers,
-                                                  (void *) __LINE__);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
-  t_op[0] = GNUNET_TESTBED_peer_get_information (peers[0],
-                                                 GNUNET_TESTBED_PIT_IDENTITY,
-                                                 &pi_cb,
-                                                 (void *) 0L);
-  t_op[1] = GNUNET_TESTBED_peer_get_information (peers[num_peers - 1],
-                                                 GNUNET_TESTBED_PIT_IDENTITY,
-                                                 &pi_cb,
-                                                 (void *) 1L);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "requested peer ids\n");
-}
-
-
-/**
- * Main: start test
- */
-int
-main (int argc, char *argv[])
-{
-  struct GNUNET_MQ_MessageHandler handlers[] = {
-    GNUNET_MQ_hd_var_size (data,
-                           GNUNET_MESSAGE_TYPE_DUMMY,
-                           struct GNUNET_MessageHeader,
-                           NULL),
-    GNUNET_MQ_handler_end ()
-  };
-
-  initialized = GNUNET_NO;
-  static const struct GNUNET_HashCode *ports[2];
-  const char *config_file;
-  char port_id[] = "test port";
-
-  GNUNET_CRYPTO_hash (port_id, sizeof (port_id), &port);
-
-  GNUNET_log_setup ("test", "DEBUG", NULL);
-  config_file = "test_cadet.conf";
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Start\n");
-
-  /* Find out requested size */
-  if (strstr (argv[0], "_2_") != NULL)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "DIRECT CONNECTIONs\n");
-    peers_requested = 2;
-  }
-  else if (strstr (argv[0], "_5_") != NULL)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "5 PEER LINE\n");
-    peers_requested = 5;
-  }
-  else
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "SIZE UNKNOWN, USING 2\n");
-    peers_requested = 2;
-  }
-
-  /* Find out requested test */
-  if (strstr (argv[0], "_forward") != NULL)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "FORWARD\n");
-    test = FORWARD;
-    test_name = "unicast";
-    ok_goal = 4;
-  }
-  else if (strstr (argv[0], "_signal") != NULL)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "SIGNAL\n");
-    test = P2P_SIGNAL;
-    test_name = "signal";
-    ok_goal = 4;
-  }
-  else if (strstr (argv[0], "_speed_ack") != NULL)
-  {
-    /* Test is supposed to generate the following callbacks:
-     * 1 incoming channel (@dest)
-     * TOTAL_PACKETS received data packet (@dest)
-     * TOTAL_PACKETS received data packet (@orig)
-     * 1 received channel destroy (@dest)
-     */
-    ok_goal = TOTAL_PACKETS * 2 + 2;
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "SPEED_ACK\n");
-    test = SPEED_ACK;
-    test_name = "speed ack";
-  }
-  else if (strstr (argv[0], "_speed") != NULL)
-  {
-    /* Test is supposed to generate the following callbacks:
-     * 1 incoming channel (@dest)
-     * 1 initial packet (@dest)
-     * TOTAL_PACKETS received data packet (@dest)
-     * 1 received data packet (@orig)
-     * 1 received channel destroy (@dest)
-     */
-    ok_goal = TOTAL_PACKETS + 4;
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "SPEED\n");
-    if (strstr (argv[0], "_reliable") != NULL)
-    {
-      test = SPEED_REL;
-      test_name = "speed reliable";
-      config_file = "test_cadet_drop.conf";
-    }
-    else
-    {
-      test = SPEED;
-      test_name = "speed";
-    }
-  }
-  else if (strstr (argv[0], "_keepalive") != NULL)
-  {
-    test = KEEPALIVE;
-    /* Test is supposed to generate the following callbacks:
-     * 1 incoming channel (@dest)
-     * [wait]
-     * 1 received channel destroy (@dest)
-     */
-    ok_goal = 2;
-  }
-  else
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "UNKNOWN\n");
-    test = SETUP;
-    ok_goal = 0;
-  }
-
-  if (strstr (argv[0], "backwards") != NULL)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "BACKWARDS (LEAF TO ROOT)\n");
-    test_backwards = GNUNET_YES;
-    GNUNET_asprintf (&test_name, "backwards %s", test_name);
-  }
-
-  p_ids = 0;
-  ports[0] = &port;
-  ports[1] = NULL;
-  GNUNET_CADET_TEST_ruN ("test_cadet_small",
-                         config_file,
-                         peers_requested,
-                         &tmain,
-                         NULL,        /* tmain cls */
-                         &connect_handler,
-                         NULL,
-                         &disconnect_handler,
-                         handlers,
-                         ports);
-  if (NULL != strstr (argv[0], "_reliable"))
-    msg_dropped = 0;            /* dropped should be retransmitted */
-
-  if (ok_goal > ok - msg_dropped)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "FAILED! (%d/%d)\n", ok, ok_goal);
-    return 1;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "success\n");
-  return 0;
-}
-
-/* end of test_cadet.c */
diff --git a/src/cadet/test_cadet_single.c b/src/cadet/test_cadet_single.c
deleted file mode 100644
index b45b0af5d..000000000
--- a/src/cadet/test_cadet_single.c
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file cadet/test_cadet_single.c
- * @brief test cadet single: test of cadet channels with just one client
- * @author Bartlomiej Polot
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_dht_service.h"
-#include "gnunet_testing_lib.h"
-#include "gnunet_cadet_service.h"
-
-#define REPETITIONS 5
-#define DATA_SIZE 35000
-
-struct GNUNET_TESTING_Peer *me;
-
-static struct GNUNET_CADET_Handle *cadet;
-
-static struct GNUNET_CADET_Channel *ch1;
-
-static struct GNUNET_CADET_Channel *ch2;
-
-static int result;
-
-static struct GNUNET_SCHEDULER_Task *abort_task;
-
-static struct GNUNET_SCHEDULER_Task *connect_task;
-
-static unsigned int repetition;
-
-static struct GNUNET_CADET_TransmitHandle *nth;
-
-static struct GNUNET_CADET_Port *port;
-
-
-/* forward declaration */
-static size_t
-do_send (void *cls, size_t size, void *buf);
-
-
-/**
- * Shutdown nicely
- */
-static void
-do_shutdown (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "shutdown\n");
-  if (NULL != port)
-  {
-    GNUNET_CADET_close_port (port);
-    port = NULL;
-  }
-  if (NULL != nth)
-  {
-    GNUNET_CADET_notify_transmit_ready_cancel (nth);
-    nth = NULL;
-  }
-  if (NULL != abort_task)
-  {
-    GNUNET_SCHEDULER_cancel (abort_task);
-    abort_task = NULL;
-  }
-  if (NULL != connect_task)
-  {
-    GNUNET_SCHEDULER_cancel (connect_task);
-    connect_task = NULL;
-  }
-  if (NULL != ch1)
-  {
-    GNUNET_CADET_channel_destroy (ch1);
-    ch1 = NULL;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Disconnect clients\n");
-  if (NULL != cadet)
-  {
-    GNUNET_CADET_disconnect (cadet);
-    cadet = NULL;
-  }
-  else
-  {
-    GNUNET_break (0);
-  }
-}
-
-
-/**
- * Something went wrong and timed out. Kill everything and set error flag
- */
-static void
-do_abort (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ABORT\n");
-  result = GNUNET_SYSERR;
-  abort_task = NULL;
-  GNUNET_SCHEDULER_shutdown ();
-}
-
-
-/**
- * Function is called whenever a message is received.
- *
- * @param cls closure (set from GNUNET_CADET_connect)
- * @param channel connection to the other end
- * @param channel_ctx place to store local state associated with the channel
- * @param message the actual message
- * @return #GNUNET_OK to keep the connection open,
- *         #GNUNET_SYSERR to close it (signal serious error)
- */
-static int
-data_callback (void *cls,
-               struct GNUNET_CADET_Channel *channel,
-               void **channel_ctx,
-               const struct GNUNET_MessageHeader *message)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Data callback! Repetition %u/%u\n",
-              repetition, REPETITIONS);
-  repetition++;
-  if (repetition < REPETITIONS)
-  {
-    struct GNUNET_CADET_Channel *my_channel;
-    if (0 == repetition % 2)
-      my_channel = ch1;
-    else
-      my_channel = ch2;
-    nth = GNUNET_CADET_notify_transmit_ready (my_channel,
-                                              GNUNET_NO,
-                                              GNUNET_TIME_UNIT_FOREVER_REL,
-                                              sizeof (struct GNUNET_MessageHeader)
-                                              + DATA_SIZE,
-                                              &do_send, NULL);
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "All data OK. Destroying channel.\n");
-  GNUNET_assert (NULL == nth);
-  GNUNET_CADET_channel_destroy (ch1);
-  ch1 = NULL;
-  return GNUNET_OK;
-}
-
-
-/**
- * Method called whenever another peer has added us to a channel
- * the other peer initiated.
- *
- * @param cls closure
- * @param channel new handle to the channel
- * @param initiator peer that started the channel
- * @param port port number
- * @param options channel option flags
- * @return initial channel context for the channel
- *         (can be NULL -- that's not an error)
- */
-static void *
-inbound_channel (void *cls,
-                 struct GNUNET_CADET_Channel *channel,
-                 const struct GNUNET_PeerIdentity *initiator,
-                 const struct GNUNET_HashCode *port,
-                 enum GNUNET_CADET_ChannelOption options)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "received incoming channel on port %s\n",
-              GNUNET_h2s (port));
-  ch2 = channel;
-  return NULL;
-}
-
-
-/**
- * Function called whenever an inbound channel is destroyed.  Should clean up
- * any associated state.
- *
- * @param cls closure (set from GNUNET_CADET_connect)
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
- */
-static void
-channel_end (void *cls,
-             const struct GNUNET_CADET_Channel *channel,
-             void *channel_ctx)
-{
-  long id = (long) cls;
-
-  nth = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "incoming channel closed at peer %ld\n",
-              id);
-  if ( (REPETITIONS == repetition) &&
-       (channel == ch2) )
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "everything fine! finishing!\n");
-    result = GNUNET_OK;
-    GNUNET_SCHEDULER_shutdown ();
-  }
-  if (channel == ch2)
-    ch2 = NULL;
-  if (channel == ch1)
-    ch1 = NULL;
-}
-
-
-/**
- * Handler array for traffic received on peer1
- */
-static struct GNUNET_CADET_MessageHandler handlers1[] = {
-  {&data_callback, 1, 0},
-  {NULL, 0, 0}
-};
-
-
-/**
- * Data send callback: fillbuffer with test packet.
- *
- * @param cls Closure (unused).
- * @param size Buffer size.
- * @param buf Buffer to fill.
- * @return size of test packet.
- */
-static size_t
-do_send (void *cls, size_t size, void *buf)
-{
-  struct GNUNET_MessageHeader *m = buf;
-
-  nth = NULL;
-  if (NULL == buf)
-  {
-    GNUNET_break (0);
-    result = GNUNET_SYSERR;
-    return 0;
-  }
-  m->size = htons (sizeof (struct GNUNET_MessageHeader) + DATA_SIZE);
-  m->type = htons (1);
-  memset (&m[1], 0, DATA_SIZE);
-  GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader) + DATA_SIZE);
-  return sizeof (struct GNUNET_MessageHeader) + DATA_SIZE;
-}
-
-/**
- * Connect to other client and send data
- *
- * @param cls Closue (unused).
- */
-static void
-do_connect (void *cls)
-{
-  struct GNUNET_PeerIdentity id;
-  size_t size = sizeof (struct GNUNET_MessageHeader) + DATA_SIZE;
-
-  connect_task = NULL;
-  GNUNET_TESTING_peer_get_identity (me, &id);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "CONNECT BY PORT\n");
-  ch1 = GNUNET_CADET_channel_create (cadet, NULL, &id, GC_u2h (1),
-                                     GNUNET_CADET_OPTION_DEFAULT);
-  nth = GNUNET_CADET_notify_transmit_ready (ch1,
-                                            GNUNET_NO,
-                                            GNUNET_TIME_UNIT_FOREVER_REL,
-                                            size,
-                                            &do_send,
-                                            NULL);
-}
-
-
-/**
- * Initialize framework and start test
- *
- * @param cls Closure (unused).
- * @param cfg Configuration handle.
- * @param peer Testing peer handle.
- */
-static void
-run (void *cls,
-     const struct GNUNET_CONFIGURATION_Handle *cfg,
-     struct GNUNET_TESTING_Peer *peer)
-{
-  me = peer;
-  GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
-  abort_task =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                    (GNUNET_TIME_UNIT_SECONDS, 15), &do_abort,
-                                    NULL);
-  cadet = GNUNET_CADET_connect (cfg,       /* configuration */
-                              (void *) 1L,     /* cls */
-                              &channel_end,      /* inbound end hndlr */
-                              handlers1); /* traffic handlers */
-  port = GNUNET_CADET_open_port (cadet,
-                                 GC_u2h (1),
-                                 &inbound_channel,
-                                 (void *) 1L);
-
-
-  if (NULL == cadet)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Couldn't connect to cadet :(\n");
-    result = GNUNET_SYSERR;
-    return;
-  }
-  connect_task
-    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
-                                    &do_connect,
-                                    NULL);
-}
-
-
-/**
- * Main
- */
-int
-main (int argc,
-      char *argv[])
-{
-  result = GNUNET_NO;
-  if (0 != GNUNET_TESTING_peer_run ("test-cadet-local",
-                                    "test_cadet.conf",
-                                    &run, NULL))
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "run failed\n");
-    return 2;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Final result: %d\n",
-              result);
-  return (result == GNUNET_OK) ? 0 : 1;
-}
-
-/* end of test_cadet_single.c */
diff --git a/src/consensus/Makefile.am b/src/consensus/Makefile.am
index 2b1987fbc..c0205ee5d 100644
--- a/src/consensus/Makefile.am
+++ b/src/consensus/Makefile.am
@@ -18,8 +18,6 @@ if USE_COVERAGE
   AM_CFLAGS = -fprofile-arcs -ftest-coverage
 endif
 
-bin_PROGRAMS = \
- gnunet-consensus-profiler
 
 libexec_PROGRAMS = \
  gnunet-service-consensus
@@ -86,6 +84,9 @@ libgnunet_plugin_block_consensus_la_LDFLAGS = \
 
 
 if HAVE_TESTING
+bin_PROGRAMS = \
+ gnunet-consensus-profiler
+
 check_PROGRAMS = \
  test_consensus_api
 
diff --git a/src/consensus/consensus_protocol.h b/src/consensus/consensus_protocol.h
index e0002de56..f2933ed6f 100644
--- a/src/consensus/consensus_protocol.h
+++ b/src/consensus/consensus_protocol.h
@@ -45,37 +45,37 @@ GNUNET_NETWORK_STRUCT_BEGIN
 struct GNUNET_CONSENSUS_RoundContextMessage
 {
   /**
-   * Type: GNUNET_MESSAGE_TYPE_CONSENSUS_P2P_ROUND_CONTEXT
+   * Type: #GNUNET_MESSAGE_TYPE_CONSENSUS_P2P_ROUND_CONTEXT
    */
   struct GNUNET_MessageHeader header;
 
   /**
    * A value from 'enum PhaseKind'.
    */
-  uint16_t kind;
+  uint16_t kind GNUNET_PACKED;
 
   /**
    * Number of the first peer
    * in canonical order.
    */
-  int16_t peer1;
+  int16_t peer1 GNUNET_PACKED;
 
   /**
    * Number of the second peer in canonical order.
    */
-  int16_t peer2;
+  int16_t peer2 GNUNET_PACKED;
 
   /**
    * Repetition of the gradecast phase.
    */
-  int16_t repetition;
+  int16_t repetition GNUNET_PACKED;
 
   /**
    * Leader in the gradecast phase.
    *
    * Can be different from both peer1 and peer2.
    */
-  int16_t leader;
+  int16_t leader GNUNET_PACKED;
 
   /**
    * Non-zero if this set reconciliation
@@ -85,7 +85,7 @@ struct GNUNET_CONSENSUS_RoundContextMessage
    *
    * Ignored for set operations that are not within gradecasts.
    */
-  uint16_t is_contested;
+  uint16_t is_contested GNUNET_PACKED;
 };
 
 
@@ -104,7 +104,7 @@ struct ConsensusElement
    * Payload element_type, only valid
    * if this is not a marker element.
    */
-  uint16_t payload_type;
+  uint16_t payload_type GNUNET_PACKED;
 
   /**
    * Is this a marker element?
@@ -117,7 +117,7 @@ struct ConsensusElement
 
 struct ConsensusSizeElement
 {
-  struct ConsensusElement ce GNUNET_PACKED;
+  struct ConsensusElement ce;
 
   uint64_t size GNUNET_PACKED;
   uint8_t sender_index;
@@ -125,7 +125,7 @@ struct ConsensusSizeElement
 
 struct ConsensusStuffedElement
 {
-  struct ConsensusElement ce GNUNET_PACKED;
+  struct ConsensusElement ce;
   struct GNUNET_HashCode rand GNUNET_PACKED;
 };
 
diff --git a/src/consensus/gnunet-consensus-profiler.c b/src/consensus/gnunet-consensus-profiler.c
index 65542f4cd..8cc1b3512 100644
--- a/src/consensus/gnunet-consensus-profiler.c
+++ b/src/consensus/gnunet-consensus-profiler.c
@@ -515,31 +515,55 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char **argv)
 {
-   static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-      { 'n', "num-peers", NULL,
-        gettext_noop ("number of peers in consensus"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_peers },
-      { 'k', "value-replication", NULL,
-        gettext_noop ("how many peers (random selection without replacement) receive one value?"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &replication },
-      { 'x', "num-values", NULL,
-        gettext_noop ("number of values"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_values },
-      { 't', "timeout", NULL,
-        gettext_noop ("consensus timeout"),
-        GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &conclude_timeout },
-      { 'd', "delay", NULL,
-        gettext_noop ("delay until consensus starts"),
-        GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &consensus_delay },
-      { 's', "statistics", NULL,
-        gettext_noop ("write statistics to file"),
-        GNUNET_YES, &GNUNET_GETOPT_set_filename, &statistics_filename },
-      { 'S', "dist-static", NULL,
-        gettext_noop ("distribute elements to a static subset of good peers"),
-        GNUNET_YES, &GNUNET_GETOPT_set_one, &dist_static },
-      { 'V', "verbose", NULL,
-        gettext_noop ("be more verbose (print received values)"),
-        GNUNET_NO, &GNUNET_GETOPT_set_one, &verbose },
+   struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('n',
+                                     "num-peers",
+                                     NULL,
+                                     gettext_noop ("number of peers in consensus"),
+                                     &num_peers),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('k',
+                                     "value-replication",
+                                     NULL,
+                                     gettext_noop ("how many peers (random selection without replacement) receive one value?"),
+                                     &replication),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('x',
+                                     "num-values",
+                                     NULL,
+                                     gettext_noop ("number of values"),
+                                     &num_values),
+
+      GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('t',
+                                              "timeout",
+                                              NULL,
+                                              gettext_noop ("consensus timeout"),
+                                              &conclude_timeout),
+
+
+      GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('d',
+                                              "delay",
+                                              NULL,
+                                              gettext_noop ("delay until consensus starts"),
+                                              &consensus_delay),
+
+      GNUNET_GETOPT_OPTION_FILENAME ('s',
+                                     "statistics",
+                                     "FILENAME",
+                                     gettext_noop ("write statistics to file"),
+                                     &statistics_filename),
+
+      GNUNET_GETOPT_OPTION_SET_ONE ('S',
+                                    "dist-static",
+                                    gettext_noop ("distribute elements to a static subset of good peers"),
+                                    &dist_static),
+
+      GNUNET_GETOPT_OPTION_SET_ONE ('V',
+                                    "verbose",
+                                    gettext_noop ("be more verbose (print received values)"),
+                                    &verbose),
+
       GNUNET_GETOPT_OPTION_END
   };
   conclude_timeout = GNUNET_TIME_UNIT_SECONDS;
diff --git a/src/consensus/gnunet-service-consensus.c b/src/consensus/gnunet-service-consensus.c
index b934f468f..4af7199aa 100644
--- a/src/consensus/gnunet-service-consensus.c
+++ b/src/consensus/gnunet-service-consensus.c
@@ -393,6 +393,14 @@ struct DiffEntry
   struct GNUNET_CONTAINER_MultiHashMap *changes;
 };
 
+struct SetHandle
+{
+  struct SetHandle *prev;
+  struct SetHandle *next;
+
+  struct GNUNET_SET_Handle *h;
+};
+
 
 
 /**
@@ -499,6 +507,9 @@ struct ConsensusSession
    * Bounded Eppstein lower bound.
    */
   uint64_t lower_bound;
+
+  struct SetHandle *set_handles_head;
+  struct SetHandle *set_handles_tail;
 };
 
 /**
@@ -894,7 +905,7 @@ cmp_uint64_t (const void *pa, const void *pb)
  * in the result set.
  *
  * @param cls closure
- * @param element a result element, only valid if status is GNUNET_SET_STATUS_OK
+ * @param element a result element, only valid if status is #GNUNET_SET_STATUS_OK
  * @param current_size current set size
  * @param status see enum GNUNET_SET_Status
  */
@@ -1011,6 +1022,7 @@ set_result_cb (void *cls,
                     "P%u: lower bound %llu\n",
                     session->local_peer_idx,
                     (long long) session->lower_bound);
+        GNUNET_free (copy);
       }
       return;
     }
@@ -1329,7 +1341,10 @@ commit_set (struct ConsensusSession *session,
   if (PHASE_KIND_ALL_TO_ALL_2 == task->key.kind)
   {
     struct GNUNET_SET_Element element;
-    struct ConsensusSizeElement cse = { 0 };
+    struct ConsensusSizeElement cse = {
+      .size = 0,
+      .sender_index = 0
+    };
     GNUNET_log (GNUNET_ERROR_TYPE_INFO, "inserting size marker\n");
     cse.ce.marker = CONSENSUS_MARKER_SIZE;
     cse.size = GNUNET_htonll (session->first_size);
@@ -1382,7 +1397,10 @@ commit_set (struct ConsensusSession *session,
         for (i = 0; i < evil.num; i++)
         {
           struct GNUNET_SET_Element element;
-          struct ConsensusStuffedElement se = { 0 };
+          struct ConsensusStuffedElement se = {
+            .ce.payload_type = 0,
+            .ce.marker = 0,
+          };
           element.data = &se;
           element.size = sizeof (struct ConsensusStuffedElement);
           element.element_type = GNUNET_BLOCK_TYPE_CONSENSUS_ELEMENT;
@@ -1663,6 +1681,12 @@ set_copy_cb (void *cls, struct GNUNET_SET_Handle *copy)
   struct TaskEntry *task = scc->task;
   struct SetKey dst_set_key = scc->dst_set_key;
   struct SetEntry *set;
+  struct SetHandle *sh = GNUNET_new (struct SetHandle);
+
+  sh->h = copy;
+  GNUNET_CONTAINER_DLL_insert (task->step->session->set_handles_head,
+                               task->step->session->set_handles_tail,
+                               sh);
 
   GNUNET_free (scc);
   set = GNUNET_new (struct SetEntry);
@@ -2105,7 +2129,7 @@ task_start_reconcile (struct TaskEntry *task)
 
   if (task->key.peer1 == session->local_peer_idx)
   {
-    struct GNUNET_CONSENSUS_RoundContextMessage rcm = { 0 };
+    struct GNUNET_CONSENSUS_RoundContextMessage rcm;
 
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "P%u: Looking up set {%s} to run remote union\n",
@@ -2120,6 +2144,7 @@ task_start_reconcile (struct TaskEntry *task)
     rcm.peer2 = htons (task->key.peer2);
     rcm.leader = htons (task->key.leader);
     rcm.repetition = htons (task->key.repetition);
+    rcm.is_contested = htons (0);
 
     GNUNET_assert (NULL == setop->op);
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: initiating set op with P%u, our set is %s\n",
@@ -3152,6 +3177,11 @@ handle_client_join (void *cls,
     client_set = GNUNET_new (struct SetEntry);
     client_set->h = GNUNET_SET_create (cfg,
                                        GNUNET_SET_OPERATION_UNION);
+    struct SetHandle *sh = GNUNET_new (struct SetHandle);
+    sh->h = client_set->h;
+    GNUNET_CONTAINER_DLL_insert (session->set_handles_head,
+                                 session->set_handles_tail,
+                                 sh);
     client_set->key = ((struct SetKey) { SET_KIND_CURRENT, 0, 0 });
     put_set (session,
              client_set);
@@ -3370,6 +3400,14 @@ client_disconnect_cb (void *cls,
   GNUNET_CONTAINER_DLL_remove (sessions_head,
                                sessions_tail,
                                session);
+
+  while (session->set_handles_head)
+  {
+    struct SetHandle *sh = session->set_handles_head;
+    session->set_handles_head = sh->next;
+    GNUNET_SET_destroy (sh->h);
+    GNUNET_free (sh);
+  }
   GNUNET_free (session);
 }
 
diff --git a/src/consensus/test_consensus.conf b/src/consensus/test_consensus.conf
index f78b77d09..881251a66 100644
--- a/src/consensus/test_consensus.conf
+++ b/src/consensus/test_consensus.conf
@@ -5,7 +5,7 @@ GNUNET_TEST_HOME = /tmp/test-consensus/
 #OPTIONS = -L INFO
 BINARY = gnunet-service-evil-consensus
 
-#PREFIX = valgrind
+PREFIX = valgrind
 
 #EVIL_SPEC = 0;cram-all;noreplace;5
 #EVIL_SPEC = 0;cram;5/1;cram;5
diff --git a/src/conversation/Makefile.am b/src/conversation/Makefile.am
index 83313e7f8..cc2938144 100644
--- a/src/conversation/Makefile.am
+++ b/src/conversation/Makefile.am
@@ -180,7 +180,7 @@ gnunet_service_conversation_LDADD = \
   libgnunetconversation.la \
   libgnunetspeaker.la \
   libgnunetmicrophone.la \
-  $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(INTLLIBS)
 gnunet_service_conversation_LDFLAGS = \
diff --git a/src/conversation/gnunet-conversation.c b/src/conversation/gnunet-conversation.c
index 925db4665..c5275c0de 100644
--- a/src/conversation/gnunet-conversation.c
+++ b/src/conversation/gnunet-conversation.c
@@ -1265,13 +1265,20 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'e', "ego", "NAME",
-     gettext_noop ("sets the NAME of the ego to use for the phone (and name resolution)"),
-     1, &GNUNET_GETOPT_set_string, &ego_name},
-    {'p', "phone", "LINE",
-      gettext_noop ("sets the LINE to use for the phone"),
-     1, &GNUNET_GETOPT_set_string, &line},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_STRING ('e',
+                                 "ego",
+                                 "NAME",
+                                 gettext_noop ("sets the NAME of the ego to use for the phone (and name resolution)"),
+                                 &ego_name),
+
+    GNUNET_GETOPT_OPTION_STRING ('p',
+                                 "phone",
+                                 "LINE",
+                                 gettext_noop ("sets the LINE to use for the phone"),
+                                 &line),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/conversation/gnunet-helper-audio-playback.c b/src/conversation/gnunet-helper-audio-playback.c
index e965cb2aa..4344e1d41 100644
--- a/src/conversation/gnunet-helper-audio-playback.c
+++ b/src/conversation/gnunet-helper-audio-playback.c
@@ -549,7 +549,6 @@ ogg_demux_and_decode ()
  */
 static int
 stdin_receiver (void *cls,
-                void *client,
                 const struct GNUNET_MessageHeader *msg)
 {
   struct AudioMessage *audio;
@@ -727,12 +726,14 @@ ogg_init ()
   ogg_sync_init (&oy);
 }
 
+
 static void
 drain_callback (pa_stream*s, int success, void *userdata)
 {
   pa_threaded_mainloop_signal (m, 0);
 }
 
+
 /**
  * The main function for the playback helper.
  *
@@ -746,7 +747,7 @@ main (int argc, char *argv[])
   static unsigned long long toff;
 
   char readbuf[MAXLINE];
-  struct GNUNET_SERVER_MessageStreamTokenizer *stdin_mst;
+  struct GNUNET_MessageStreamTokenizer *stdin_mst;
   char c;
   ssize_t ret;
 #ifdef DEBUG_READ_PURE_OGG
@@ -762,7 +763,7 @@ main (int argc, char *argv[])
     GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "pipe");
     return 1;
   }
-  stdin_mst = GNUNET_SERVER_mst_create (&stdin_receiver, NULL);
+  stdin_mst = GNUNET_MST_create (&stdin_receiver, NULL);
   ogg_init ();
   pa_init ();
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -802,11 +803,11 @@ main (int argc, char *argv[])
     }
     else
 #endif
-    GNUNET_SERVER_mst_receive (stdin_mst, NULL,
-                               readbuf, ret,
-                               GNUNET_NO, GNUNET_NO);
+    GNUNET_MST_from_buffer (stdin_mst,
+                            readbuf, ret,
+                            GNUNET_NO, GNUNET_NO);
   }
-  GNUNET_SERVER_mst_destroy (stdin_mst);
+  GNUNET_MST_destroy (stdin_mst);
   if (stream_out)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
diff --git a/src/conversation/gnunet-service-conversation.c b/src/conversation/gnunet-service-conversation.c
index f80cc1d11..5f43bfe80 100644
--- a/src/conversation/gnunet-service-conversation.c
+++ b/src/conversation/gnunet-service-conversation.c
@@ -779,7 +779,6 @@ handle_cadet_hangup_message (void *cls,
   {
   case CS_CALLEE_INIT:
     GNUNET_break_op (0);
-    destroy_line_cadet_channels (ch);
     return;
   case CS_CALLEE_RINGING:
   case CS_CALLEE_CONNECTED:
@@ -1110,7 +1109,7 @@ handle_client_call_message (void *cls,
                                line->channel_tail,
                                ch);
   ch->status = CS_CALLER_CALLING;
-  ch->channel = GNUNET_CADET_channel_creatE (cadet,
+  ch->channel = GNUNET_CADET_channel_create (cadet,
                                              ch,
                                              &msg->target,
                                              &msg->line_port,
@@ -1264,7 +1263,7 @@ handle_client_register_message (void *cls,
   };
 
   line->line_port = msg->line_port;
-  line->port = GNUNET_CADET_open_porT (cadet,
+  line->port = GNUNET_CADET_open_port (cadet,
                                        &msg->line_port,
                                        &inbound_channel,
                                        line,
@@ -1307,7 +1306,7 @@ run (void *cls,
   GNUNET_assert (GNUNET_OK ==
                  GNUNET_CRYPTO_get_peer_identity (cfg,
                                                   &my_identity));
-  cadet = GNUNET_CADET_connecT (cfg);
+  cadet = GNUNET_CADET_connect (cfg);
   if (NULL == cadet)
   {
     GNUNET_break (0);
diff --git a/src/conversation/microphone.c b/src/conversation/microphone.c
index 94f52f8dc..7871433a3 100644
--- a/src/conversation/microphone.c
+++ b/src/conversation/microphone.c
@@ -64,13 +64,11 @@ struct Microphone
  * Function to process the audio from the record helper
  *
  * @param cls clsoure with our `struct Microphone`
- * @param client NULL
  * @param msg the message from the helper
  * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
  */
 static int
 process_record_messages (void *cls,
-                         void *client,
                          const struct GNUNET_MessageHeader *msg)
 {
   struct Microphone *mic = cls;
diff --git a/src/core/core_api.c b/src/core/core_api.c
index fd789295d..ed8ce364d 100644
--- a/src/core/core_api.c
+++ b/src/core/core_api.c
@@ -321,7 +321,7 @@ core_mq_send_impl (struct GNUNET_MQ_Handle *mq,
 
   /* check message size for sanity */
   msize = ntohs (msg->size);
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct SendMessage))
+  if (msize >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct SendMessage))
   {
     GNUNET_break (0);
     GNUNET_MQ_impl_send_continue (mq);
@@ -796,7 +796,7 @@ GNUNET_CORE_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
   h->handlers = GNUNET_MQ_copy_handlers (handlers);
   h->hcnt = GNUNET_MQ_count_handlers (handlers);
   GNUNET_assert (h->hcnt <
-                 (GNUNET_SERVER_MAX_MESSAGE_SIZE -
+                 (GNUNET_MAX_MESSAGE_SIZE -
                   sizeof (struct InitMessage)) / sizeof (uint16_t));
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Connecting to CORE service\n");
diff --git a/src/core/gnunet-core.c b/src/core/gnunet-core.c
index d91dc304d..ed89b1946 100644
--- a/src/core/gnunet-core.c
+++ b/src/core/gnunet-core.c
@@ -171,10 +171,11 @@ main (int argc,
       char *const *argv)
 {
   int res;
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'m', "monitor", NULL,
-     gettext_noop ("provide information about all current connections (continuously)"),
-     0, &GNUNET_GETOPT_set_one, &monitor_connections},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('m',
+                                  "monitor",
+                                  gettext_noop ("provide information about all current connections (continuously)"),
+                                  &monitor_connections),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/core/gnunet-service-core.c b/src/core/gnunet-service-core.c
index 31b91f12f..625bf9655 100644
--- a/src/core/gnunet-service-core.c
+++ b/src/core/gnunet-service-core.c
@@ -807,7 +807,7 @@ GSC_CLIENTS_deliver_message (const struct GNUNET_PeerIdentity *sender,
 {
   size_t size = msize + sizeof (struct NotifyTrafficMessage);
 
-  if (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
diff --git a/src/core/gnunet-service-core_kx.c b/src/core/gnunet-service-core_kx.c
index 906898512..8a7cada5c 100644
--- a/src/core/gnunet-service-core_kx.c
+++ b/src/core/gnunet-service-core_kx.c
@@ -263,6 +263,11 @@ struct GSC_KeyExchangeInfo
   struct GNUNET_MQ_Handle *mq;
 
   /**
+   * Our message stream tokenizer (for encrypted payload).
+   */
+  struct GNUNET_MessageStreamTokenizer *mst;
+
+  /**
    * PING message we transmit to the other peer.
    */
   struct PingMessage ping;
@@ -370,11 +375,6 @@ static struct GNUNET_CRYPTO_EcdhePrivateKey *my_ephemeral_key;
 static struct EphemeralKeyMessage current_ekm;
 
 /**
- * Our message stream tokenizer (for encrypted payload).
- */
-static struct GNUNET_SERVER_MessageStreamTokenizer *mst;
-
-/**
  * DLL head.
  */
 static struct GSC_KeyExchangeInfo *kx_head;
@@ -702,6 +702,55 @@ setup_fresh_ping (struct GSC_KeyExchangeInfo *kx)
 
 
 /**
+ * Deliver P2P message to interested clients.  Invokes send twice,
+ * once for clients that want the full message, and once for clients
+ * that only want the header
+ *
+ * @param cls the `struct GSC_KeyExchangeInfo`
+ * @param m the message
+ */
+static int
+deliver_message (void *cls,
+                 const struct GNUNET_MessageHeader *m)
+{
+  struct GSC_KeyExchangeInfo *kx = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Decrypted message of type %d from %s\n",
+              ntohs (m->type),
+              GNUNET_i2s (kx->peer));
+  if (GNUNET_CORE_KX_STATE_UP != kx->status)
+  {
+    GNUNET_STATISTICS_update (GSC_stats,
+                              gettext_noop ("# PAYLOAD dropped (out of order)"),
+                              1,
+                              GNUNET_NO);
+    return GNUNET_OK;
+  }
+  switch (ntohs (m->type))
+  {
+  case GNUNET_MESSAGE_TYPE_CORE_BINARY_TYPE_MAP:
+  case GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP:
+    GSC_SESSIONS_set_typemap (kx->peer, m);
+    return GNUNET_OK;
+  case GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP:
+    GSC_SESSIONS_confirm_typemap (kx->peer, m);
+    return GNUNET_OK;
+  default:
+    GSC_CLIENTS_deliver_message (kx->peer,
+                                 m,
+                                 ntohs (m->size),
+                                 GNUNET_CORE_OPTION_SEND_FULL_INBOUND);
+    GSC_CLIENTS_deliver_message (kx->peer,
+                                 m,
+                                 sizeof (struct GNUNET_MessageHeader),
+                                 GNUNET_CORE_OPTION_SEND_HDR_INBOUND);
+  }
+  return GNUNET_OK;
+}
+
+
+/**
  * Function called by transport to notify us that
  * a peer connected to us (on the network level).
  * Starts the key exchange with the given peer.
@@ -727,6 +776,8 @@ handle_transport_notify_connect (void *cls,
                             1,
                             GNUNET_NO);
   kx = GNUNET_new (struct GSC_KeyExchangeInfo);
+  kx->mst = GNUNET_MST_create (&deliver_message,
+                               kx);
   kx->mq = mq;
   kx->peer = pid;
   kx->set_key_retry_frequency = INITIAL_SET_KEY_RETRY_FREQUENCY;
@@ -801,6 +852,7 @@ handle_transport_notify_disconnect (void *cls,
   GNUNET_CONTAINER_DLL_remove (kx_head,
                                kx_tail,
                                kx);
+  GNUNET_MST_destroy (kx->mst);
   GNUNET_free (kx);
 }
 
@@ -1417,24 +1469,6 @@ GSC_KX_encrypt_and_transmit (struct GSC_KeyExchangeInfo *kx,
 
 
 /**
- * Closure for #deliver_message()
- */
-struct DeliverMessageContext
-{
-
-  /**
-   * Key exchange context.
-   */
-  struct GSC_KeyExchangeInfo *kx;
-
-  /**
-   * Sender of the message.
-   */
-  const struct GNUNET_PeerIdentity *peer;
-};
-
-
-/**
  * We received an encrypted message.  Check that it is
  * well-formed (size-wise).
  *
@@ -1475,7 +1509,6 @@ handle_encrypted (void *cls,
   struct GNUNET_TIME_Absolute t;
   struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
   struct GNUNET_CRYPTO_AuthKey auth_key;
-  struct DeliverMessageContext dmc;
   uint16_t size = ntohs (m->header.size);
   char buf[size] GNUNET_ALIGN;
 
@@ -1620,15 +1653,12 @@ handle_encrypted (void *cls,
                             gettext_noop ("# bytes of payload decrypted"),
                             size - sizeof (struct EncryptedMessage),
                             GNUNET_NO);
-  dmc.kx = kx;
-  dmc.peer = kx->peer;
   if (GNUNET_OK !=
-      GNUNET_SERVER_mst_receive (mst,
-                                 &dmc,
-                                 &buf[sizeof (struct EncryptedMessage)],
-                                 size - sizeof (struct EncryptedMessage),
-                                 GNUNET_YES,
-                                 GNUNET_NO))
+      GNUNET_MST_from_buffer (kx->mst,
+                              &buf[sizeof (struct EncryptedMessage)],
+                              size - sizeof (struct EncryptedMessage),
+                              GNUNET_YES,
+                              GNUNET_NO))
     GNUNET_break_op (0);
 }
 
@@ -1656,57 +1686,6 @@ handle_transport_notify_excess_bw (void *cls,
 
 
 /**
- * Deliver P2P message to interested clients.  Invokes send twice,
- * once for clients that want the full message, and once for clients
- * that only want the header
- *
- * @param cls always NULL
- * @param client who sent us the message (struct GSC_KeyExchangeInfo)
- * @param m the message
- */
-static int
-deliver_message (void *cls,
-                 void *client,
-                 const struct GNUNET_MessageHeader *m)
-{
-  struct DeliverMessageContext *dmc = client;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Decrypted message of type %d from %s\n",
-              ntohs (m->type),
-              GNUNET_i2s (dmc->peer));
-  if (GNUNET_CORE_KX_STATE_UP != dmc->kx->status)
-  {
-    GNUNET_STATISTICS_update (GSC_stats,
-                              gettext_noop ("# PAYLOAD dropped (out of order)"),
-                              1,
-                              GNUNET_NO);
-    return GNUNET_OK;
-  }
-  switch (ntohs (m->type))
-  {
-  case GNUNET_MESSAGE_TYPE_CORE_BINARY_TYPE_MAP:
-  case GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP:
-    GSC_SESSIONS_set_typemap (dmc->peer, m);
-    return GNUNET_OK;
-  case GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP:
-    GSC_SESSIONS_confirm_typemap (dmc->peer, m);
-    return GNUNET_OK;
-  default:
-    GSC_CLIENTS_deliver_message (dmc->peer,
-                                 m,
-                                 ntohs (m->size),
-                                 GNUNET_CORE_OPTION_SEND_FULL_INBOUND);
-    GSC_CLIENTS_deliver_message (dmc->peer,
-                                 m,
-                                 sizeof (struct GNUNET_MessageHeader),
-                                 GNUNET_CORE_OPTION_SEND_HDR_INBOUND);
-  }
-  return GNUNET_OK;
-}
-
-
-/**
  * Setup the message that links the ephemeral key to our persistent
  * public key and generate the appropriate signature.
  */
@@ -1829,8 +1808,6 @@ GSC_KX_init (struct GNUNET_CRYPTO_EddsaPrivateKey *pk)
   rekey_task = GNUNET_SCHEDULER_add_delayed (REKEY_FREQUENCY,
                                              &do_rekey,
                                              NULL);
-  mst = GNUNET_SERVER_mst_create (&deliver_message,
-                                  NULL);
   transport
     = GNUNET_TRANSPORT_core_connect (GSC_cfg,
                                      &GSC_my_identity,
@@ -1874,11 +1851,6 @@ GSC_KX_done ()
     GNUNET_free (my_private_key);
     my_private_key = NULL;
   }
-  if (NULL != mst)
-  {
-    GNUNET_SERVER_mst_destroy (mst);
-    mst = NULL;
-  }
   if (NULL != nc)
   {
     GNUNET_notification_context_destroy (nc);
diff --git a/src/datacache/Makefile.am b/src/datacache/Makefile.am
index 431b3179e..670a64926 100644
--- a/src/datacache/Makefile.am
+++ b/src/datacache/Makefile.am
@@ -53,6 +53,7 @@ libgnunet_plugin_datacache_sqlite_la_SOURCES = \
   plugin_datacache_sqlite.c
 libgnunet_plugin_datacache_sqlite_la_LIBADD = \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
+  $(top_builddir)/src/sq/libgnunetsq.la \
   $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
   $(LTLIBINTL)
 libgnunet_plugin_datacache_sqlite_la_LDFLAGS = \
diff --git a/src/datacache/plugin_datacache_sqlite.c b/src/datacache/plugin_datacache_sqlite.c
index 5567077d3..dd79d0125 100644
--- a/src/datacache/plugin_datacache_sqlite.c
+++ b/src/datacache/plugin_datacache_sqlite.c
@@ -26,6 +26,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_datacache_plugin.h"
+#include "gnunet_sq_lib.h"
 #include <sqlite3.h>
 
 #define LOG(kind,...) GNUNET_log_from (kind, "datacache-sqlite", __VA_ARGS__)
@@ -60,6 +61,41 @@ struct Plugin
   char *fn;
 
   /**
+   * Prepared statement for #sqlite_plugin_put.
+   */
+  sqlite3_stmt *insert_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_get.
+   */
+  sqlite3_stmt *get_count_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_get.
+   */
+  sqlite3_stmt *get_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_del.
+   */
+  sqlite3_stmt *del_select_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_del.
+   */
+  sqlite3_stmt *del_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_get_random.
+   */
+  sqlite3_stmt *get_random_stmt;
+
+  /**
+   * Prepared statement for #sqlite_plugin_get_closest.
+   */
+  sqlite3_stmt *get_closest_stmt;
+
+  /**
    * Number of key-value pairs in the database.
    */
   unsigned int num_items;
@@ -132,60 +168,47 @@ sqlite_plugin_put (void *cls,
                    const struct GNUNET_PeerIdentity *path_info)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
-  int64_t dval;
+  uint32_t type32 = type;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint32 (&type32),
+    GNUNET_SQ_query_param_absolute_time (&discard_time),
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_fixed_size (data, size),
+    GNUNET_SQ_query_param_fixed_size (path_info,
+                                      path_info_len * sizeof (struct GNUNET_PeerIdentity)),
+    GNUNET_SQ_query_param_end
+  };
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Processing PUT of %u bytes with key `%4s' and expiration %s\n",
+       "Processing PUT of %u bytes with key `%s' and expiration %s\n",
        (unsigned int) size,
        GNUNET_h2s (key),
-       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (discard_time), GNUNET_YES));
-  dval = (int64_t) discard_time.abs_value_us;
-  if (dval < 0)
-    dval = INT64_MAX;
-  if (sq_prepare
-      (plugin->dbh,
-       "INSERT INTO ds090 (type, expire, key, value, path) VALUES (?, ?, ?, ?, ?)",
-       &stmt) != SQLITE_OK)
-  {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
-    return -1;
-  }
-  if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, dval)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_blob (stmt, 3,
-                          key, sizeof (struct GNUNET_HashCode),
-                          SQLITE_TRANSIENT)) ||
-      (SQLITE_OK != sqlite3_bind_blob (stmt, 4,
-                                       data, size,
-                                       SQLITE_TRANSIENT)) ||
-      (SQLITE_OK != sqlite3_bind_blob (stmt, 5,
-                                       path_info,
-                                       path_info_len * sizeof (struct GNUNET_PeerIdentity),
-                                       SQLITE_TRANSIENT)))
+       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (discard_time),
+                                               GNUNET_YES));
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->insert_stmt,
+                      params))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_xxx");
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->insert_stmt);
     return -1;
   }
-  if (SQLITE_DONE != sqlite3_step (stmt))
+  if (SQLITE_DONE !=
+      sqlite3_step (plugin->insert_stmt))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->insert_stmt);
     return -1;
   }
   plugin->num_items++;
-  if (SQLITE_OK != sqlite3_finalize (stmt))
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_finalize");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->insert_stmt);
   return size + OVERHEAD;
 }
 
@@ -209,120 +232,119 @@ sqlite_plugin_get (void *cls,
                    void *iter_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
+  uint32_t type32 = type;
   struct GNUNET_TIME_Absolute now;
   struct GNUNET_TIME_Absolute exp;
-  unsigned int size;
-  const char *dat;
+  size_t size;
+  void *dat;
   unsigned int cnt;
-  unsigned int off;
+  uint32_t off;
   unsigned int total;
-  unsigned int psize;
-  char scratch[256];
-  int64_t ntime;
-  const struct GNUNET_PeerIdentity *path;
+  size_t psize;
+  struct GNUNET_PeerIdentity *path;
+  struct GNUNET_SQ_QueryParam params_count[] = {
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_uint32 (&type32),
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_QueryParam params_select[] = {
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_uint32 (&type32),
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_uint32 (&off),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_variable_size (&dat,
+                                         &size),
+    GNUNET_SQ_result_spec_absolute_time (&exp),
+    GNUNET_SQ_result_spec_variable_size ((void **) &path,
+                                         &psize),
+    GNUNET_SQ_result_spec_end
+  };
 
   now = GNUNET_TIME_absolute_get ();
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Processing GET for key `%4s'\n",
+       "Processing GET for key `%s'\n",
        GNUNET_h2s (key));
-  if (sq_prepare
-      (plugin->dbh,
-       "SELECT count(*) FROM ds090 WHERE key=? AND type=? AND expire >= ?",
-       &stmt) != SQLITE_OK)
-  {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
-    return 0;
-  }
-  ntime = (int64_t) now.abs_value_us;
-  GNUNET_assert (ntime >= 0);
-  if ((SQLITE_OK !=
-       sqlite3_bind_blob (stmt, 1, key, sizeof (struct GNUNET_HashCode),
-                          SQLITE_TRANSIENT)) ||
-      (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 3, now.abs_value_us)))
+
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->get_count_stmt,
+                      params_count))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_xxx");
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_count_stmt);
     return 0;
   }
-
-  if (SQLITE_ROW != sqlite3_step (stmt))
+  if (SQLITE_ROW !=
+      sqlite3_step (plugin->get_count_stmt))
   {
     LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite_step");
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_count_stmt);
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "No content found when processing GET for key `%4s'\n",
+         "No content found when processing GET for key `%s'\n",
          GNUNET_h2s (key));
     return 0;
   }
-  total = sqlite3_column_int (stmt, 0);
-  sqlite3_finalize (stmt);
-  if ((0 == total) || (NULL == iter))
+  total = sqlite3_column_int (plugin->get_count_stmt,
+                              0);
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->get_count_stmt);
+  if ( (0 == total) ||
+       (NULL == iter) )
   {
     if (0 == total)
       LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "No content found when processing GET for key `%4s'\n",
+           "No content found when processing GET for key `%s'\n",
            GNUNET_h2s (key));
     return total;
   }
 
   cnt = 0;
-  off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, total);
+  off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                  total);
   while (cnt < total)
   {
     off = (off + 1) % total;
-    GNUNET_snprintf (scratch, sizeof (scratch),
-                     "SELECT value,expire,path FROM ds090 WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET %u",
-                     off);
-    if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
-    {
-      LOG_SQLITE (plugin->dbh,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sq_prepare");
-      return cnt;
-    }
-    if ((SQLITE_OK !=
-         sqlite3_bind_blob (stmt, 1,
-                            key,
-                            sizeof (struct GNUNET_HashCode),
-                            SQLITE_TRANSIENT)) ||
-        (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
-        (SQLITE_OK != sqlite3_bind_int64 (stmt, 3, now.abs_value_us)))
+    if (GNUNET_OK !=
+        GNUNET_SQ_bind (plugin->get_stmt,
+                        params_select))
     {
       LOG_SQLITE (plugin->dbh,
                   GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                   "sqlite3_bind_xxx");
-      sqlite3_finalize (stmt);
+      GNUNET_SQ_reset (plugin->dbh,
+                       plugin->get_stmt);
       return cnt;
     }
-    if (sqlite3_step (stmt) != SQLITE_ROW)
+    if (SQLITE_ROW !=
+        sqlite3_step (plugin->get_stmt))
       break;
-    size = sqlite3_column_bytes (stmt, 0);
-    dat = sqlite3_column_blob (stmt, 0);
-    exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
-    psize = sqlite3_column_bytes (stmt, 2);
+    if (GNUNET_OK !=
+        GNUNET_SQ_extract_result (plugin->get_stmt,
+                                  rs))
+    {
+      GNUNET_break (0);
+      GNUNET_SQ_reset (plugin->dbh,
+                       plugin->get_stmt);
+      break;
+    }
     if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
     {
       GNUNET_break (0);
       psize = 0;
+      path = NULL;
     }
     psize /= sizeof (struct GNUNET_PeerIdentity);
-    if (0 != psize)
-      path = sqlite3_column_blob (stmt, 2);
-    else
-      path = NULL;
-    ntime = (int64_t) exp.abs_value_us;
-    if (ntime == INT64_MAX)
-      exp = GNUNET_TIME_UNIT_FOREVER_ABS;
     cnt++;
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Found %u-byte result when processing GET for key `%4s'\n",
+         "Found %u-byte result when processing GET for key `%s'\n",
          (unsigned int) size,
          GNUNET_h2s (key));
     if (GNUNET_OK != iter (iter_cls,
@@ -334,11 +356,17 @@ sqlite_plugin_get (void *cls,
                            psize,
                            path))
     {
-      sqlite3_finalize (stmt);
+      GNUNET_SQ_cleanup_result (rs);
+      GNUNET_SQ_reset (plugin->dbh,
+                       plugin->get_stmt);
       break;
     }
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_cleanup_result (rs);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_stmt);
   }
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->get_stmt);
   return cnt;
 }
 
@@ -354,79 +382,73 @@ static int
 sqlite_plugin_del (void *cls)
 {
   struct Plugin *plugin = cls;
-  unsigned long long rowid;
-  unsigned int dsize;
-  sqlite3_stmt *stmt;
-  sqlite3_stmt *dstmt;
+  uint64_t rowid;
+  void *data;
+  size_t dsize;
   struct GNUNET_HashCode hc;
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_uint64 (&rowid),
+    GNUNET_SQ_result_spec_auto_from_type (&hc),
+    GNUNET_SQ_result_spec_variable_size ((void **) &data,
+                                         &dsize),
+    GNUNET_SQ_result_spec_end
+  };
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint64 (&rowid),
+    GNUNET_SQ_query_param_end
+  };
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Processing DEL\n");
-  stmt = NULL;
-  dstmt = NULL;
-  if (SQLITE_OK !=
-      sq_prepare (plugin->dbh,
-                  "SELECT _ROWID_,key,value FROM ds090 ORDER BY expire ASC LIMIT 1",
-                  &stmt))
-  {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
-    if (stmt != NULL)
-      (void) sqlite3_finalize (stmt);
-    return GNUNET_SYSERR;
-  }
-  if (SQLITE_ROW != sqlite3_step (stmt))
+  if (SQLITE_ROW !=
+      sqlite3_step (plugin->del_select_stmt))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-    (void) sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->del_select_stmt);
     return GNUNET_SYSERR;
   }
-  rowid = sqlite3_column_int64 (stmt, 0);
-  GNUNET_assert (sqlite3_column_bytes (stmt, 1) == sizeof (struct GNUNET_HashCode));
-  GNUNET_memcpy (&hc, sqlite3_column_blob (stmt, 1), sizeof (struct GNUNET_HashCode));
-  dsize = sqlite3_column_bytes (stmt, 2);
-  if (SQLITE_OK != sqlite3_finalize (stmt))
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_step");
-  if (SQLITE_OK !=
-      sq_prepare (plugin->dbh,
-                  "DELETE FROM ds090 WHERE _ROWID_=?", &dstmt))
+  if (GNUNET_OK !=
+      GNUNET_SQ_extract_result (plugin->del_select_stmt,
+                                rs))
   {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
-    if (stmt != NULL)
-      (void) sqlite3_finalize (stmt);
+    GNUNET_break (0);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->del_select_stmt);
     return GNUNET_SYSERR;
   }
-  if (SQLITE_OK != sqlite3_bind_int64 (dstmt, 1, rowid))
+  GNUNET_SQ_cleanup_result (rs);
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->del_select_stmt);
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->del_stmt,
+                      params))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind");
-    (void) sqlite3_finalize (dstmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->del_stmt);
     return GNUNET_SYSERR;
   }
-  if (SQLITE_DONE != sqlite3_step (dstmt))
+  if (SQLITE_DONE !=
+      sqlite3_step (plugin->del_stmt))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-    (void) sqlite3_finalize (dstmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->del_stmt);
     return GNUNET_SYSERR;
   }
   plugin->num_items--;
   plugin->env->delete_notify (plugin->env->cls,
                               &hc,
                               dsize + OVERHEAD);
-  if (SQLITE_OK != sqlite3_finalize (dstmt))
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_finalize");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->del_stmt);
   return GNUNET_OK;
 }
 
@@ -445,17 +467,28 @@ sqlite_plugin_get_random (void *cls,
                           void *iter_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
   struct GNUNET_TIME_Absolute exp;
-  unsigned int size;
-  const char *dat;
-  unsigned int off;
-  unsigned int psize;
-  unsigned int type;
-  char scratch[256];
-  int64_t ntime;
-  const struct GNUNET_PeerIdentity *path;
-  const struct GNUNET_HashCode *key;
+  size_t size;
+  void *dat;
+  uint32_t off;
+  size_t psize;
+  uint32_t type;
+  struct GNUNET_PeerIdentity *path;
+  struct GNUNET_HashCode key;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint32 (&off),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_variable_size (&dat,
+                                         &size),
+    GNUNET_SQ_result_spec_absolute_time (&exp),
+    GNUNET_SQ_result_spec_variable_size ((void **) &path,
+                                         &psize),
+    GNUNET_SQ_result_spec_auto_from_type (&key),
+    GNUNET_SQ_result_spec_uint32 (&type),
+    GNUNET_SQ_result_spec_end
+  };
 
   if (0 == plugin->num_items)
     return 0;
@@ -463,60 +496,51 @@ sqlite_plugin_get_random (void *cls,
     return 1;
   off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
                                   plugin->num_items);
-  GNUNET_snprintf (scratch,
-                   sizeof (scratch),
-                   "SELECT value,expire,path,key,type FROM ds090 ORDER BY key LIMIT 1 OFFSET %u",
-                   off);
-  if (SQLITE_OK !=
-      sq_prepare (plugin->dbh, scratch, &stmt))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->get_random_stmt,
+                      params))
   {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
     return 0;
   }
-  if (SQLITE_ROW != sqlite3_step (stmt))
+  if (SQLITE_ROW !=
+      sqlite3_step (plugin->get_random_stmt))
+  {
+    GNUNET_break (0);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_random_stmt);
+    return 0;
+  }
+  if (GNUNET_OK !=
+      GNUNET_SQ_extract_result (plugin->get_random_stmt,
+                                rs))
   {
     GNUNET_break (0);
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_random_stmt);
     return 0;
   }
-  size = sqlite3_column_bytes (stmt, 0);
-  dat = sqlite3_column_blob (stmt, 0);
-  exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
-  psize = sqlite3_column_bytes (stmt, 2);
   if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
   {
     GNUNET_break (0);
     psize = 0;
+    path = NULL;
   }
   psize /= sizeof (struct GNUNET_PeerIdentity);
-  if (0 != psize)
-    path = sqlite3_column_blob (stmt, 2);
-  else
-    path = NULL;
-
-  GNUNET_assert (sizeof (struct GNUNET_HashCode) ==
-                 sqlite3_column_bytes (stmt, 3));
-  key = sqlite3_column_blob (stmt, 3);
-  type = sqlite3_column_int (stmt, 4);
-
-  ntime = (int64_t) exp.abs_value_us;
-  if (ntime == INT64_MAX)
-    exp = GNUNET_TIME_UNIT_FOREVER_ABS;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Found %u-byte result with key %s when processing GET-RANDOM\n",
        (unsigned int) size,
-       GNUNET_h2s (key));
+       GNUNET_h2s (&key));
   (void) iter (iter_cls,
-               key,
+               &key,
                size,
                dat,
-               type,
+               (enum GNUNET_BLOCK_Type) type,
                exp,
                psize,
                path);
-  sqlite3_finalize (stmt);
+  GNUNET_SQ_cleanup_result (rs);
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->get_random_stmt);
   return 1;
 }
 
@@ -542,83 +566,73 @@ sqlite_plugin_get_closest (void *cls,
                            void *iter_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
+  uint32_t num_results32 = num_results;
   struct GNUNET_TIME_Absolute now;
   struct GNUNET_TIME_Absolute exp;
-  unsigned int size;
-  const char *dat;
+  size_t size;
+  void *dat;
   unsigned int cnt;
-  unsigned int psize;
-  unsigned int type;
-  int64_t ntime;
-  const struct GNUNET_PeerIdentity *path;
+  size_t psize;
+  uint32_t type;
+  struct GNUNET_HashCode hc;
+  struct GNUNET_PeerIdentity *path;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_uint32 (&num_results32),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_variable_size (&dat,
+                                         &size),
+    GNUNET_SQ_result_spec_absolute_time (&exp),
+    GNUNET_SQ_result_spec_variable_size ((void **) &path,
+                                         &psize),
+    GNUNET_SQ_result_spec_uint32 (&type),
+    GNUNET_SQ_result_spec_auto_from_type (&hc),
+    GNUNET_SQ_result_spec_end
+  };
 
   now = GNUNET_TIME_absolute_get ();
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Processing GET_CLOSEST for key `%4s'\n",
+       "Processing GET_CLOSEST for key `%s'\n",
        GNUNET_h2s (key));
-  if (SQLITE_OK !=
-      sq_prepare (plugin->dbh,
-                  "SELECT value,expire,path,type,key FROM ds090 WHERE key>=? AND expire >= ? ORDER BY KEY ASC LIMIT ?",
-                  &stmt))
-  {
-    LOG_SQLITE (plugin->dbh,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sq_prepare");
-    return 0;
-  }
-  ntime = (int64_t) now.abs_value_us;
-  GNUNET_assert (ntime >= 0);
-  if ((SQLITE_OK !=
-       sqlite3_bind_blob (stmt,
-                          1,
-                          key,
-                          sizeof (struct GNUNET_HashCode),
-                          SQLITE_TRANSIENT)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, now.abs_value_us)) ||
-      (SQLITE_OK != sqlite3_bind_int (stmt, 3, num_results)) )
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->get_closest_stmt,
+                      params))
   {
     LOG_SQLITE (plugin->dbh,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_xxx");
-    sqlite3_finalize (stmt);
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->get_closest_stmt);
     return 0;
   }
   cnt = 0;
-  while (SQLITE_ROW == sqlite3_step (stmt))
+  while (SQLITE_ROW ==
+         sqlite3_step (plugin->get_closest_stmt))
   {
-    if (sizeof (struct GNUNET_HashCode) !=
-        sqlite3_column_bytes (stmt, 4))
+    if (GNUNET_OK !=
+        GNUNET_SQ_extract_result (plugin->get_closest_stmt,
+                                  rs))
     {
       GNUNET_break (0);
       break;
     }
-    size = sqlite3_column_bytes (stmt, 0);
-    dat = sqlite3_column_blob (stmt, 0);
-    exp.abs_value_us = sqlite3_column_int64 (stmt, 1);
-    psize = sqlite3_column_bytes (stmt, 2);
-    type = sqlite3_column_int (stmt, 3);
-    key = sqlite3_column_blob (stmt, 4);
     if (0 != psize % sizeof (struct GNUNET_PeerIdentity))
     {
       GNUNET_break (0);
       psize = 0;
+      path = NULL;
     }
     psize /= sizeof (struct GNUNET_PeerIdentity);
-    if (0 != psize)
-      path = sqlite3_column_blob (stmt, 2);
-    else
-      path = NULL;
-    ntime = (int64_t) exp.abs_value_us;
-    if (ntime == INT64_MAX)
-      exp = GNUNET_TIME_UNIT_FOREVER_ABS;
     cnt++;
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Found %u-byte result at %s when processing GET_CLOSE\n",
          (unsigned int) size,
-         GNUNET_h2s (key));
+         GNUNET_h2s (&hc));
     if (GNUNET_OK != iter (iter_cls,
-                           key,
+                           &hc,
                            size,
                            dat,
                            type,
@@ -626,11 +640,13 @@ sqlite_plugin_get_closest (void *cls,
                            psize,
                            path))
     {
-      sqlite3_finalize (stmt);
+      GNUNET_SQ_cleanup_result (rs);
       break;
     }
+    GNUNET_SQ_cleanup_result (rs);
   }
-  sqlite3_finalize (stmt);
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->get_closest_stmt);
   return cnt;
 }
 
@@ -703,6 +719,51 @@ libgnunet_plugin_datacache_sqlite_init (void *cls)
   plugin->env = env;
   plugin->dbh = dbh;
   plugin->fn = fn_utf8;
+
+  if ( (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "INSERT INTO ds090 (type, expire, key, value, path) "
+                    "VALUES (?, ?, ?, ?, ?)",
+                    &plugin->insert_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT count(*) FROM ds090 "
+                    "WHERE key=? AND type=? AND expire >= ?",
+                    &plugin->get_count_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT value,expire,path FROM ds090 "
+                    "WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET ?",
+                    &plugin->get_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT _ROWID_,key,value FROM ds090 ORDER BY expire ASC LIMIT 1",
+                    &plugin->del_select_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "DELETE FROM ds090 WHERE _ROWID_=?",
+                    &plugin->del_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT value,expire,path,key,type FROM ds090 "
+                    "ORDER BY key LIMIT 1 OFFSET ?",
+                    &plugin->get_random_stmt)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT value,expire,path,type,key FROM ds090 "
+                    "WHERE key>=? AND expire >= ? ORDER BY KEY ASC LIMIT ?",
+                    &plugin->get_closest_stmt))
+       )
+  {
+    LOG_SQLITE (plugin->dbh,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                "sq_prepare");
+    GNUNET_break (SQLITE_OK ==
+                  sqlite3_close (plugin->dbh));
+    GNUNET_free (plugin);
+    return NULL;
+  }
+
   api = GNUNET_new (struct GNUNET_DATACACHE_PluginFunctions);
   api->cls = plugin;
   api->get = &sqlite_plugin_get;
@@ -741,6 +802,13 @@ libgnunet_plugin_datacache_sqlite_done (void *cls)
                        plugin->fn);
   GNUNET_free_non_null (plugin->fn);
 #endif
+  sqlite3_finalize (plugin->insert_stmt);
+  sqlite3_finalize (plugin->get_count_stmt);
+  sqlite3_finalize (plugin->get_stmt);
+  sqlite3_finalize (plugin->del_select_stmt);
+  sqlite3_finalize (plugin->del_stmt);
+  sqlite3_finalize (plugin->get_random_stmt);
+  sqlite3_finalize (plugin->get_closest_stmt);
   result = sqlite3_close (plugin->dbh);
 #if SQLITE_VERSION_NUMBER >= 3007000
   if (SQLITE_BUSY == result)
diff --git a/src/datastore/Makefile.am b/src/datastore/Makefile.am
index 1285020de..9b8cf365f 100644
--- a/src/datastore/Makefile.am
+++ b/src/datastore/Makefile.am
@@ -115,6 +115,7 @@ noinst_LTLIBRARIES = \
 libgnunet_plugin_datastore_sqlite_la_SOURCES = \
   plugin_datastore_sqlite.c
 libgnunet_plugin_datastore_sqlite_la_LIBADD = \
+  $(top_builddir)/src/sq/libgnunetsq.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
   $(LTLIBINTL)
diff --git a/src/datastore/datastore.h b/src/datastore/datastore.h
index 9de72f064..5fd360161 100644
--- a/src/datastore/datastore.h
+++ b/src/datastore/datastore.h
@@ -119,9 +119,14 @@ struct GetKeyMessage
   uint32_t type GNUNET_PACKED;
 
   /**
-   * Offset of the result.
+   * UID at which to start the search
    */
-  uint64_t offset GNUNET_PACKED;
+  uint64_t next_uid GNUNET_PACKED;
+
+  /**
+   * If true return a random result
+   */
+  uint32_t random GNUNET_PACKED;
 
   /**
    * Desired key.
@@ -148,9 +153,14 @@ struct GetMessage
   uint32_t type GNUNET_PACKED;
 
   /**
-   * Offset of the result.
+   * UID at which to start the search
+   */
+  uint64_t next_uid GNUNET_PACKED;
+
+  /**
+   * If true return a random result
    */
-  uint64_t offset GNUNET_PACKED;
+  uint32_t random GNUNET_PACKED;
 
 };
 
@@ -172,9 +182,9 @@ struct GetZeroAnonymityMessage
   uint32_t type GNUNET_PACKED;
 
   /**
-   * Offset of the result.
+   * UID at which to start the search
    */
-  uint64_t offset GNUNET_PACKED;
+  uint64_t next_uid GNUNET_PACKED;
 
 };
 
diff --git a/src/datastore/datastore_api.c b/src/datastore/datastore_api.c
index 916e6acae..26e1e501d 100644
--- a/src/datastore/datastore_api.c
+++ b/src/datastore/datastore_api.c
@@ -33,6 +33,8 @@
 
 #define LOG(kind,...) GNUNET_log_from (kind, "datastore-api",__VA_ARGS__)
 
+#define DELAY_WARN_TIMEOUT GNUNET_TIME_UNIT_MINUTES
+
 /**
  * Collect an instane number of statistics?  May cause excessive IPC.
  */
@@ -138,6 +140,12 @@ struct GNUNET_DATASTORE_QueueEntry
   struct GNUNET_MQ_Envelope *env;
 
   /**
+   * Task we run if this entry stalls the queue and we
+   * need to warn the user.
+   */
+  struct GNUNET_SCHEDULER_Task *delay_warn_task;
+
+  /**
    * Priority in the queue.
    */
   unsigned int priority;
@@ -269,11 +277,36 @@ free_queue_entry (struct GNUNET_DATASTORE_QueueEntry *qe)
   h->queue_size--;
   if (NULL != qe->env)
     GNUNET_MQ_discard (qe->env);
+  if (NULL != qe->delay_warn_task)
+    GNUNET_SCHEDULER_cancel (qe->delay_warn_task);
   GNUNET_free (qe);
 }
 
 
 /**
+ * Task that logs an error after some time.
+ *
+ * @param qe `struct GNUNET_DATASTORE_QueueEntry` about which the error is
+ */
+static void
+delay_warning (void *cls)
+{
+  struct GNUNET_DATASTORE_QueueEntry *qe = cls;
+
+  qe->delay_warn_task = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+              "Request %p of type %u at head of datastore queue for more than %s\n",
+              qe,
+              (unsigned int) qe->response_type,
+              GNUNET_STRINGS_relative_time_to_string (DELAY_WARN_TIMEOUT,
+                                                      GNUNET_YES));
+  qe->delay_warn_task = GNUNET_SCHEDULER_add_delayed (DELAY_WARN_TIMEOUT,
+                                                      &delay_warning,
+                                                      qe);
+}
+
+
+/**
  * Handle error in sending drop request to datastore.
  *
  * @param cls closure with the datastore handle
@@ -290,8 +323,14 @@ mq_error_handler (void *cls,
        "MQ error, reconnecting to DATASTORE\n");
   do_disconnect (h);
   qe = h->queue_head;
-  if ( (NULL != qe) &&
-       (NULL == qe->env) )
+  if (NULL == qe)
+    return;
+  if (NULL != qe->delay_warn_task)
+  {
+    GNUNET_SCHEDULER_cancel (qe->delay_warn_task);
+    qe->delay_warn_task = NULL;
+  }
+  if (NULL == qe->env)
   {
     union QueueContext qc = qe->qc;
     uint16_t rt = qe->response_type;
@@ -594,6 +633,10 @@ process_queue (struct GNUNET_DATASTORE_Handle *h)
          "Not connected\n");
     return;
   }
+  GNUNET_assert (NULL == qe->delay_warn_task);
+  qe->delay_warn_task = GNUNET_SCHEDULER_add_delayed (DELAY_WARN_TIMEOUT,
+                                                      &delay_warning,
+                                                      qe);
   GNUNET_MQ_send (h->mq,
                   qe->env);
   qe->env = NULL;
@@ -958,7 +1001,7 @@ GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h,
   struct DataMessage *dm;
   union QueueContext qc;
 
-  if (size + sizeof (*dm) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (size + sizeof (*dm) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return NULL;
@@ -1169,7 +1212,7 @@ GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h,
   struct GNUNET_MQ_Envelope *env;
   union QueueContext qc;
 
-  if (sizeof (*dm) + size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (sizeof (*dm) + size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return NULL;
@@ -1282,10 +1325,7 @@ GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
  * Get a single zero-anonymity value from the datastore.
  *
  * @param h handle to the datastore
- * @param offset offset of the result (modulo num-results); set to
- *               a random 64-bit value initially; then increment by
- *               one each time; detect that all results have been found by uid
- *               being again the first uid ever returned.
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param queue_priority ranking of this request in the priority queue
  * @param max_queue_size at what queue size should this request be dropped
  *        (if other requests of higher priority are in the queue)
@@ -1299,7 +1339,7 @@ GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h,
  */
 struct GNUNET_DATASTORE_QueueEntry *
 GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
-                                     uint64_t offset,
+                                     uint64_t next_uid,
                                      unsigned int queue_priority,
                                      unsigned int max_queue_size,
                                      enum GNUNET_BLOCK_Type type,
@@ -1314,13 +1354,12 @@ GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
   GNUNET_assert (NULL != proc);
   GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Asked to get %llu-th zero-anonymity entry of type %d\n",
-       (unsigned long long) offset,
+       "Asked to get a zero-anonymity entry of type %d\n",
        type);
   env = GNUNET_MQ_msg (m,
                        GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY);
   m->type = htonl ((uint32_t) type);
-  m->offset = GNUNET_htonll (offset);
+  m->next_uid = GNUNET_htonll (next_uid);
   qc.rc.proc = proc;
   qc.rc.proc_cls = proc_cls;
   qe = make_queue_entry (h,
@@ -1349,10 +1388,8 @@ GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
  * will only be called once.
  *
  * @param h handle to the datastore
- * @param offset offset of the result (modulo num-results); set to
- *               a random 64-bit value initially; then increment by
- *               one each time; detect that all results have been found by uid
- *               being again the first uid ever returned.
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key maybe NULL (to match all entries)
  * @param type desired type, 0 for any
  * @param queue_priority ranking of this request in the priority queue
@@ -1366,7 +1403,8 @@ GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
  */
 struct GNUNET_DATASTORE_QueueEntry *
 GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
-                          uint64_t offset,
+                          uint64_t next_uid,
+                          bool random,
                           const struct GNUNET_HashCode *key,
                           enum GNUNET_BLOCK_Type type,
                           unsigned int queue_priority,
@@ -1390,14 +1428,16 @@ GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
     env = GNUNET_MQ_msg (gm,
                          GNUNET_MESSAGE_TYPE_DATASTORE_GET);
     gm->type = htonl (type);
-    gm->offset = GNUNET_htonll (offset);
+    gm->next_uid = GNUNET_htonll (next_uid);
+    gm->random = random;
   }
   else
   {
     env = GNUNET_MQ_msg (gkm,
                          GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY);
     gkm->type = htonl (type);
-    gkm->offset = GNUNET_htonll (offset);
+    gkm->next_uid = GNUNET_htonll (next_uid);
+    gkm->random = random;
     gkm->key = *key;
   }
   qc.rc.proc = proc;
diff --git a/src/datastore/gnunet-datastore.c b/src/datastore/gnunet-datastore.c
index b3d14c43c..c93bc8dd3 100644
--- a/src/datastore/gnunet-datastore.c
+++ b/src/datastore/gnunet-datastore.c
@@ -171,7 +171,7 @@ static void
 do_get ()
 {
   qe = GNUNET_DATASTORE_get_key (db_src,
-                                 offset,
+                                 0, false,
                                  NULL, GNUNET_BLOCK_TYPE_ANY,
                                  0, 1,
                                  &do_put, NULL);
@@ -239,10 +239,12 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 's', "sourcecfg", "FILENAME",
-      gettext_noop ("specifies the configuration to use to access an alternative datastore; will merge that datastore into our current datastore"),
-      1, &GNUNET_GETOPT_set_filename, &alternative_cfg },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_FILENAME ('s',
+                                   "sourcecfg",
+                                   "FILENAME",
+                                   gettext_noop ("specifies the configuration to use to access an alternative datastore; will merge that datastore into our current datastore"),
+                                       &alternative_cfg),
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
index 1e699fea3..af33c4412 100644
--- a/src/datastore/gnunet-service-datastore.c
+++ b/src/datastore/gnunet-service-datastore.c
@@ -307,7 +307,7 @@ expired_processor (void *cls,
 {
   struct GNUNET_TIME_Absolute now;
 
-  if (key == NULL)
+  if (NULL == key)
   {
     expired_kill_task =
         GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
@@ -529,7 +529,7 @@ transmit_item (void *cls,
     return GNUNET_OK;
   }
   GNUNET_assert (sizeof (struct DataMessage) + size <
-                 GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                 GNUNET_MAX_MESSAGE_SIZE);
   env = GNUNET_MQ_msg_extra (dm,
                              size,
                              GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
@@ -984,12 +984,13 @@ handle_put (void *cls,
                         size,
                         &vhash);
     plugin->api->get_key (plugin->api->cls,
-                          0,
-                          &dm->key,
-                          &vhash,
+                          0,
+                          false,
+                          &dm->key,
+                          &vhash,
                           ntohl (dm->type),
-                          &check_present,
-                          pc);
+                          &check_present,
+                          pc);
     GNUNET_SERVICE_client_continue (client);
     return;
   }
@@ -1018,7 +1019,8 @@ handle_get (void *cls,
                             1,
                             GNUNET_NO);
   plugin->api->get_key (plugin->api->cls,
-                        GNUNET_ntohll (msg->offset),
+                        GNUNET_ntohll (msg->next_uid),
+                        msg->random,
                         NULL,
                         NULL,
                         ntohl (msg->type),
@@ -1069,7 +1071,8 @@ handle_get_key (void *cls,
     return;
   }
   plugin->api->get_key (plugin->api->cls,
-                        GNUNET_ntohll (msg->offset),
+                        GNUNET_ntohll (msg->next_uid),
+                        msg->random,
                         &msg->key,
                         NULL,
                         ntohl (msg->type),
@@ -1131,7 +1134,7 @@ handle_get_zero_anonymity (void *cls,
                             1,
                             GNUNET_NO);
   plugin->api->get_zero_anonymity (plugin->api->cls,
-                                   GNUNET_ntohll (msg->offset),
+                                   GNUNET_ntohll (msg->next_uid),
                                    type,
                                    &transmit_item,
                                    client);
@@ -1241,6 +1244,7 @@ handle_remove (void *cls,
               (uint32_t) ntohl (dm->type));
   plugin->api->get_key (plugin->api->cls,
                         0,
+                        false,
                         &dm->key,
                         &vhash,
                         (enum GNUNET_BLOCK_Type) ntohl (dm->type),
diff --git a/src/datastore/plugin_datastore_heap.c b/src/datastore/plugin_datastore_heap.c
index 199c03a50..e15cacb5b 100644
--- a/src/datastore/plugin_datastore_heap.c
+++ b/src/datastore/plugin_datastore_heap.c
@@ -323,19 +323,19 @@ struct GetContext
 {
 
   /**
-   * Desired result offset / number of results.
+   * Lowest uid to consider.
    */
-  uint64_t offset;
+  uint64_t next_uid;
 
   /**
-   * The plugin.
+   * Value with lowest uid >= next_uid found so far.
    */
-  struct Plugin *plugin;
+  struct Value *value;
 
   /**
    * Requested value hash.
    */
-  const struct GNUNET_HashCode * vhash;
+  const struct GNUNET_HashCode *vhash;
 
   /**
    * Requested type.
@@ -343,68 +343,15 @@ struct GetContext
   enum GNUNET_BLOCK_Type type;
 
   /**
-   * Function to call with the result.
+   * If true, return a random value
    */
-  PluginDatumProcessor proc;
+  bool random;
 
-  /**
-   * Closure for 'proc'.
-   */
-  void *proc_cls;
 };
 
 
 /**
- * Test if a value matches the specification from the 'get' context
- *
- * @param gc query
- * @param value the value to check against the query
- * @return GNUNET_YES if the value matches
- */
-static int
-match (const struct GetContext *gc,
-       struct Value *value)
-{
-  struct GNUNET_HashCode vh;
-
-  if ( (gc->type != GNUNET_BLOCK_TYPE_ANY) &&
-       (gc->type != value->type) )
-    return GNUNET_NO;
-  if (NULL != gc->vhash)
-  {
-    GNUNET_CRYPTO_hash (&value[1], value->size, &vh);
-    if (0 != memcmp (&vh, gc->vhash, sizeof (struct GNUNET_HashCode)))
-      return GNUNET_NO;
-  }
-  return GNUNET_YES;
-}
-
-
-/**
- * Count number of matching values.
- *
- * @param cls the 'struct GetContext'
- * @param key unused
- * @param val the 'struct Value'
- * @return GNUNET_YES (continue iteration)
- */
-static int
-count_iterator (void *cls,
-                const struct GNUNET_HashCode *key,
-                void *val)
-{
-  struct GetContext *gc = cls;
-  struct Value *value = val;
-
-  if (GNUNET_NO == match (gc, value))
-    return GNUNET_OK;
-  gc->offset++;
-  return GNUNET_OK;
-}
-
-
-/**
- * Obtain matching value at 'offset'.
+ * Obtain the matching value with the lowest uid >= next_uid.
  *
  * @param cls the 'struct GetContext'
  * @param key unused
@@ -418,23 +365,29 @@ get_iterator (void *cls,
 {
   struct GetContext *gc = cls;
   struct Value *value = val;
+  struct GNUNET_HashCode vh;
 
-  if (GNUNET_NO == match (gc, value))
+  if ( (gc->type != GNUNET_BLOCK_TYPE_ANY) &&
+       (gc->type != value->type) )
     return GNUNET_OK;
-  if (0 != gc->offset--)
+  if (NULL != gc->vhash)
+  {
+    GNUNET_CRYPTO_hash (&value[1], value->size, &vh);
+    if (0 != memcmp (&vh, gc->vhash, sizeof (struct GNUNET_HashCode)))
+      return GNUNET_OK;
+  }
+  if (gc->random)
+  {
+    gc->value = value;
+    return GNUNET_NO;
+  }
+  if ( (uint64_t) (intptr_t) value < gc->next_uid)
     return GNUNET_OK;
-  if (GNUNET_NO ==
-      gc->proc (gc->proc_cls,
-                key,
-                value->size,
-                &value[1],
-                value->type,
-                value->priority,
-                value->anonymity,
-            value->expiration,
-                (uint64_t) (long) value))
-    delete_value (gc->plugin, value);
-  return GNUNET_NO;
+  if ( (NULL != gc->value) &&
+       (value > gc->value) )
+    return GNUNET_OK;
+  gc->value = value;
+  return GNUNET_OK;
 }
 
 
@@ -442,8 +395,8 @@ get_iterator (void *cls,
  * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key maybe NULL (to match all entries)
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
@@ -457,7 +410,7 @@ get_iterator (void *cls,
  * @param proc_cls closure for proc
  */
 static void
-heap_plugin_get_key (void *cls, uint64_t offset,
+heap_plugin_get_key (void *cls, uint64_t next_uid, bool random,
                      const struct GNUNET_HashCode *key,
                      const struct GNUNET_HashCode *vhash,
                      enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc,
@@ -466,25 +419,14 @@ heap_plugin_get_key (void *cls, uint64_t offset,
   struct Plugin *plugin = cls;
   struct GetContext gc;
 
-  gc.plugin = plugin;
-  gc.offset = 0;
+  gc.value = NULL;
+  gc.next_uid = next_uid;
+  gc.random = random;
   gc.vhash = vhash;
   gc.type = type;
-  gc.proc = proc;
-  gc.proc_cls = proc_cls;
   if (NULL == key)
   {
     GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
-                                           &count_iterator,
-                                           &gc);
-    if (0 == gc.offset)
-    {
-      proc (proc_cls,
-            NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-      return;
-    }
-    gc.offset = offset % gc.offset;
-    GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue,
                                            &get_iterator,
                                            &gc);
   }
@@ -492,20 +434,27 @@ heap_plugin_get_key (void *cls, uint64_t offset,
   {
     GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
                                                 key,
-                                                &count_iterator,
-                                                &gc);
-    if (0 == gc.offset)
-    {
-      proc (proc_cls,
-            NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-      return;
-    }
-    gc.offset = offset % gc.offset;
-    GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue,
-                                                key,
                                                 &get_iterator,
                                                 &gc);
   }
+  if (NULL == gc.value)
+  {
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+    return;
+  }
+  if (GNUNET_NO ==
+      proc (proc_cls,
+            &gc.value->key,
+            gc.value->size,
+            &gc.value[1],
+            gc.value->type,
+            gc.value->priority,
+            gc.value->anonymity,
+            gc.value->expiration,
+            (uint64_t) (intptr_t) gc.value))
+  {
+    delete_value (plugin, gc.value);
+  }
 }
 
 
@@ -559,7 +508,7 @@ heap_plugin_get_replication (void *cls,
             value->priority,
             value->anonymity,
             value->expiration,
-            (uint64_t) (long) value))
+            (uint64_t) (intptr_t) value))
     delete_value (plugin, value);
 }
 
@@ -595,7 +544,7 @@ heap_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
             value->priority,
             value->anonymity,
             value->expiration,
-            (uint64_t) (long) value))
+            (uint64_t) (intptr_t) value))
     delete_value (plugin, value);
 }
 
@@ -628,7 +577,7 @@ heap_plugin_update (void *cls,
 {
   struct Value *value;
 
-  value = (struct Value*) (long) uid;
+  value = (struct Value*) (intptr_t) uid;
   GNUNET_assert (NULL != value);
   if (value->expiration.abs_value_us != expire.abs_value_us)
   {
@@ -649,53 +598,43 @@ heap_plugin_update (void *cls,
  * Call the given processor on an item with zero anonymity.
  *
  * @param cls our "struct Plugin*"
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
- *        Use 0 for any type.
+ *        Must not be zero (ANY).
  * @param proc function to call on each matching value;
- *        will be called  with NULL if no value matches
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for proc
  */
 static void
-heap_plugin_get_zero_anonymity (void *cls, uint64_t offset,
+heap_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
                                 enum GNUNET_BLOCK_Type type,
                                 PluginDatumProcessor proc, void *proc_cls)
 {
   struct Plugin *plugin = cls;
   struct ZeroAnonByType *zabt;
-  struct Value *value;
-  uint64_t count;
+  struct Value *value = NULL;
 
-  count = 0;
   for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
   {
     if ( (type != GNUNET_BLOCK_TYPE_ANY) &&
-         (type != zabt->type) )
+         (type != zabt->type) )
       continue;
-    count += zabt->array_pos;
+    for (int i = 0; i < zabt->array_pos; ++i)
+    {
+      if ( (uint64_t) (intptr_t) zabt->array[i] < next_uid)
+        continue;
+      if ( (NULL != value) &&
+           (zabt->array[i] > value) )
+        continue;
+      value = zabt->array[i];
+    }
   }
-  if (0 == count)
+  if (NULL == value)
   {
     proc (proc_cls,
-          NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
+          NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
-  offset = offset % count;
-  for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next)
-  {
-    if ( (type != GNUNET_BLOCK_TYPE_ANY) &&
-         (type != zabt->type) )
-      continue;
-    if (offset >= zabt->array_pos)
-    {
-      offset -= zabt->array_pos;
-      continue;
-    }
-    break;
-  }
-  GNUNET_assert (NULL != zabt);
-  value = zabt->array[offset];
   if (GNUNET_NO ==
       proc (proc_cls,
             &value->key,
@@ -705,7 +644,7 @@ heap_plugin_get_zero_anonymity (void *cls, uint64_t offset,
             value->priority,
             value->anonymity,
             value->expiration,
-            (uint64_t) (long) value))
+            (uint64_t) (intptr_t) value))
     delete_value (plugin, value);
 }
 
diff --git a/src/datastore/plugin_datastore_mysql.c b/src/datastore/plugin_datastore_mysql.c
index 1067064aa..5ae4485cb 100644
--- a/src/datastore/plugin_datastore_mysql.c
+++ b/src/datastore/plugin_datastore_mysql.c
@@ -150,28 +150,19 @@ struct Plugin
 #define DELETE_ENTRY_BY_UID "DELETE FROM gn090 WHERE uid=?"
   struct GNUNET_MYSQL_StatementHandle *delete_entry_by_uid;
 
-#define COUNT_ENTRY_BY_HASH "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash) WHERE hash=?"
-  struct GNUNET_MYSQL_StatementHandle *count_entry_by_hash;
+#define SELECT_ENTRY "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 WHERE uid >= ? AND (rvalue >= ? OR 0 = ?) ORDER BY uid LIMIT 1"
+  struct GNUNET_MYSQL_StatementHandle *select_entry;
 
-#define SELECT_ENTRY_BY_HASH "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash) WHERE hash=? ORDER BY uid LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash) WHERE hash=? AND uid >= ? AND (rvalue >= ? OR 0 = ?) ORDER BY uid LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash;
 
-#define COUNT_ENTRY_BY_HASH_AND_VHASH "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=?"
-  struct GNUNET_MYSQL_StatementHandle *count_entry_by_hash_and_vhash;
-
-#define SELECT_ENTRY_BY_HASH_AND_VHASH "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? ORDER BY uid LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH_AND_VHASH "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? AND uid >= ? AND (rvalue >= ? OR 0 = ?) ORDER BY uid LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_vhash;
 
-#define COUNT_ENTRY_BY_HASH_AND_TYPE "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash_type_uid) WHERE hash=? AND type=?"
-  struct GNUNET_MYSQL_StatementHandle *count_entry_by_hash_and_type;
-
-#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_type_uid) WHERE hash=? AND type=? ORDER BY uid LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_type_uid) WHERE hash=? AND type=? AND uid >= ? AND (rvalue >= ? OR 0 = ?) ORDER BY uid LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_type;
 
-#define COUNT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? AND type=?"
-  struct GNUNET_MYSQL_StatementHandle *count_entry_by_hash_vhash_and_type;
-
-#define SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? AND type=? ORDER BY uid ASC LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? AND type=? AND uid >= ? AND (rvalue >= ? OR 0 = ?) ORDER BY uid LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_vhash_and_type;
 
 #define UPDATE_ENTRY "UPDATE gn090 SET prio=prio+?,expire=IF(expire>=?,expire,?) WHERE uid=?"
@@ -185,10 +176,8 @@ struct Plugin
 
 #define SELECT_IT_NON_ANONYMOUS "SELECT type,prio,anonLevel,expire,hash,value,uid "\
    "FROM gn090 FORCE INDEX (idx_anonLevel_type_rvalue) "\
-   "WHERE anonLevel=0 AND type=? AND "\
-   "(rvalue >= ? OR"\
-   "  NOT EXISTS (SELECT 1 FROM gn090 FORCE INDEX (idx_anonLevel_type_rvalue) WHERE anonLevel=0 AND type=? AND rvalue>=?)) "\
-   "ORDER BY rvalue ASC LIMIT 1"
+   "WHERE anonLevel=0 AND type=? AND uid >= ? "\
+   "ORDER BY uid LIMIT 1"
   struct GNUNET_MYSQL_StatementHandle *zero_iter;
 
 #define SELECT_IT_EXPIRATION "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_expire) WHERE expire < ? ORDER BY expire ASC LIMIT 1"
@@ -541,8 +530,8 @@ execute_select (struct Plugin *plugin,
  * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key key to match, never NULL
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
@@ -557,7 +546,8 @@ execute_select (struct Plugin *plugin,
  */
 static void
 mysql_plugin_get_key (void *cls,
-                      uint64_t offset,
+                      uint64_t next_uid,
+                      bool random,
                       const struct GNUNET_HashCode *key,
                       const struct GNUNET_HashCode *vhash,
                       enum GNUNET_BLOCK_Type type,
@@ -565,121 +555,33 @@ mysql_plugin_get_key (void *cls,
                       void *proc_cls)
 {
   struct Plugin *plugin = cls;
-  int ret;
-  uint64_t total;
-  struct GNUNET_MY_ResultSpec results_get[] = {
-    GNUNET_MY_result_spec_uint64 (&total),
-    GNUNET_MY_result_spec_end
-  };
+  uint64_t rvalue;
 
-  total = UINT64_MAX;
-  if (0 != type)
+  if (random)
   {
-    if (NULL != vhash)
-    {
-      struct GNUNET_MY_QueryParam params_get[] = {
-        GNUNET_MY_query_param_auto_from_type (key),
-        GNUNET_MY_query_param_auto_from_type (vhash),
-        GNUNET_MY_query_param_uint32 (&type),
-        GNUNET_MY_query_param_end
-      };
-
-      ret =
-        GNUNET_MY_exec_prepared (plugin->mc,
-                                 plugin->count_entry_by_hash_vhash_and_type,
-                                 params_get);
-      GNUNET_break (GNUNET_OK == ret);
-      if (GNUNET_OK == ret)
-        ret =
-          GNUNET_MY_extract_result (plugin->count_entry_by_hash_vhash_and_type,
-                                    results_get);
-      if (GNUNET_OK == ret)
-        GNUNET_break (GNUNET_NO ==
-                      GNUNET_MY_extract_result (plugin->count_entry_by_hash_vhash_and_type,
-                                                NULL));
-    }
-    else
-    {
-      struct GNUNET_MY_QueryParam params_get[] = {
-        GNUNET_MY_query_param_auto_from_type (key),
-        GNUNET_MY_query_param_uint32 (&type),
-        GNUNET_MY_query_param_end
-      };
-
-      ret =
-        GNUNET_MY_exec_prepared (plugin->mc,
-                                 plugin->count_entry_by_hash_and_type,
-                                 params_get);
-      GNUNET_break (GNUNET_OK == ret);
-      if (GNUNET_OK == ret)
-        ret =
-          GNUNET_MY_extract_result (plugin->count_entry_by_hash_and_type,
-                                    results_get);
-      if (GNUNET_OK == ret)
-        GNUNET_break (GNUNET_NO ==
-                      GNUNET_MY_extract_result (plugin->count_entry_by_hash_and_type,
-                                                NULL));
-    }
+    rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX);
+    next_uid = 0;
   }
   else
-  {
-    if (NULL != vhash)
-    {
-      struct GNUNET_MY_QueryParam params_get[] = {
-        GNUNET_MY_query_param_auto_from_type (key),
-        GNUNET_MY_query_param_auto_from_type (vhash),
-        GNUNET_MY_query_param_end
-      };
+    rvalue = 0;
 
-      ret =
-        GNUNET_MY_exec_prepared (plugin->mc,
-                                 plugin->count_entry_by_hash_and_vhash,
-                                 params_get);
-      GNUNET_break (GNUNET_OK == ret);
-      if (GNUNET_OK == ret)
-        ret =
-          GNUNET_MY_extract_result (plugin->count_entry_by_hash_and_vhash,
-                                    results_get);
-      if (GNUNET_OK == ret)
-        GNUNET_break (GNUNET_NO ==
-                      GNUNET_MY_extract_result (plugin->count_entry_by_hash_and_vhash,
-                                                NULL));
-    }
-    else
-    {
-      struct GNUNET_MY_QueryParam params_get[] = {
-        GNUNET_MY_query_param_auto_from_type (key),
-        GNUNET_MY_query_param_end
-      };
-
-      ret =
-        GNUNET_MY_exec_prepared (plugin->mc,
-                                 plugin->count_entry_by_hash,
-                                 params_get);
-      GNUNET_break (GNUNET_OK == ret);
-      if (GNUNET_OK == ret)
-        ret =
-          GNUNET_MY_extract_result (plugin->count_entry_by_hash,
-                                    results_get);
-      if (GNUNET_OK == ret)
-        GNUNET_break (GNUNET_NO ==
-                      GNUNET_MY_extract_result (plugin->count_entry_by_hash,
-                                                NULL));
-    }
-  }
-  if ( (GNUNET_OK != ret) ||
-       (0 >= total) )
+  if (NULL == key)
   {
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
+    struct GNUNET_MY_QueryParam params_select[] = {
+      GNUNET_MY_query_param_uint64 (&next_uid),
+      GNUNET_MY_query_param_uint64 (&rvalue),
+      GNUNET_MY_query_param_uint64 (&rvalue),
+      GNUNET_MY_query_param_end
+    };
+
+    execute_select (plugin,
+                    plugin->select_entry,
+                    proc,
+                    proc_cls,
+                    params_select);
   }
-  offset = offset % total;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Obtaining %llu/%lld result for GET `%s'\n",
-              (unsigned long long) offset,
-              (unsigned long long) total,
-              GNUNET_h2s (key));
-  if (type != GNUNET_BLOCK_TYPE_ANY)
+  else if (type != GNUNET_BLOCK_TYPE_ANY)
   {
     if (NULL != vhash)
     {
@@ -687,7 +589,9 @@ mysql_plugin_get_key (void *cls,
         GNUNET_MY_query_param_auto_from_type (key),
         GNUNET_MY_query_param_auto_from_type (vhash),
         GNUNET_MY_query_param_uint32 (&type),
-        GNUNET_MY_query_param_uint64 (&offset),
+        GNUNET_MY_query_param_uint64 (&next_uid),
+        GNUNET_MY_query_param_uint64 (&rvalue),
+        GNUNET_MY_query_param_uint64 (&rvalue),
         GNUNET_MY_query_param_end
       };
 
@@ -702,7 +606,9 @@ mysql_plugin_get_key (void *cls,
       struct GNUNET_MY_QueryParam params_select[] = {
         GNUNET_MY_query_param_auto_from_type (key),
         GNUNET_MY_query_param_uint32 (&type),
-        GNUNET_MY_query_param_uint64 (&offset),
+        GNUNET_MY_query_param_uint64 (&next_uid),
+        GNUNET_MY_query_param_uint64 (&rvalue),
+        GNUNET_MY_query_param_uint64 (&rvalue),
         GNUNET_MY_query_param_end
       };
 
@@ -720,7 +626,9 @@ mysql_plugin_get_key (void *cls,
       struct GNUNET_MY_QueryParam params_select[] = {
         GNUNET_MY_query_param_auto_from_type (key),
         GNUNET_MY_query_param_auto_from_type (vhash),
-        GNUNET_MY_query_param_uint64 (&offset),
+        GNUNET_MY_query_param_uint64 (&next_uid),
+        GNUNET_MY_query_param_uint64 (&rvalue),
+        GNUNET_MY_query_param_uint64 (&rvalue),
         GNUNET_MY_query_param_end
       };
 
@@ -734,7 +642,9 @@ mysql_plugin_get_key (void *cls,
     {
       struct GNUNET_MY_QueryParam params_select[] = {
         GNUNET_MY_query_param_auto_from_type (key),
-        GNUNET_MY_query_param_uint64 (&offset),
+        GNUNET_MY_query_param_uint64 (&next_uid),
+        GNUNET_MY_query_param_uint64 (&rvalue),
+        GNUNET_MY_query_param_uint64 (&rvalue),
         GNUNET_MY_query_param_end
       };
 
@@ -753,28 +663,26 @@ mysql_plugin_get_key (void *cls,
  * Get a zero-anonymity datum from the datastore.
  *
  * @param cls our `struct Plugin *`
- * @param offset offset of the result
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
- *        Use 0 for any type.
- * @param proc function to call on a matching value or NULL
+ *        Must not be zero (ANY).
+ * @param proc function to call on a matching value;
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for @a proc
  */
 static void
 mysql_plugin_get_zero_anonymity (void *cls,
-                                 uint64_t offset,
+                                 uint64_t next_uid,
                                  enum GNUNET_BLOCK_Type type,
                                  PluginDatumProcessor proc,
                                  void *proc_cls)
 {
   struct Plugin *plugin = cls;
   uint32_t typei = (uint32_t) type;
-  uint64_t rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                              UINT64_MAX);
+
   struct GNUNET_MY_QueryParam params_zero_iter[] = {
     GNUNET_MY_query_param_uint32 (&typei),
-    GNUNET_MY_query_param_uint64 (&rvalue),
-    GNUNET_MY_query_param_uint32 (&typei),
-    GNUNET_MY_query_param_uint64 (&rvalue),
+    GNUNET_MY_query_param_uint64 (&next_uid),
     GNUNET_MY_query_param_end
   };
 
@@ -1209,6 +1117,7 @@ libgnunet_plugin_datastore_mysql_init (void *cls)
        ") ENGINE=InnoDB") || MRUNS ("SET AUTOCOMMIT = 1") ||
       PINIT (plugin->insert_entry, INSERT_ENTRY) ||
       PINIT (plugin->delete_entry_by_uid, DELETE_ENTRY_BY_UID) ||
+      PINIT (plugin->select_entry, SELECT_ENTRY) ||
       PINIT (plugin->select_entry_by_hash, SELECT_ENTRY_BY_HASH) ||
       PINIT (plugin->select_entry_by_hash_and_vhash,
              SELECT_ENTRY_BY_HASH_AND_VHASH) ||
@@ -1216,13 +1125,7 @@ libgnunet_plugin_datastore_mysql_init (void *cls)
              SELECT_ENTRY_BY_HASH_AND_TYPE) ||
       PINIT (plugin->select_entry_by_hash_vhash_and_type,
              SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE) ||
-      PINIT (plugin->count_entry_by_hash, COUNT_ENTRY_BY_HASH) ||
       PINIT (plugin->get_size, SELECT_SIZE) ||
-      PINIT (plugin->count_entry_by_hash_and_vhash,
-             COUNT_ENTRY_BY_HASH_AND_VHASH) ||
-      PINIT (plugin->count_entry_by_hash_and_type, COUNT_ENTRY_BY_HASH_AND_TYPE)
-      || PINIT (plugin->count_entry_by_hash_vhash_and_type,
-                COUNT_ENTRY_BY_HASH_VHASH_AND_TYPE) ||
       PINIT (plugin->update_entry, UPDATE_ENTRY) ||
       PINIT (plugin->dec_repl, DEC_REPL) ||
       PINIT (plugin->zero_iter, SELECT_IT_NON_ANONYMOUS) ||
diff --git a/src/datastore/plugin_datastore_postgres.c b/src/datastore/plugin_datastore_postgres.c
index 7b04cc68a..0376ebb6c 100644
--- a/src/datastore/plugin_datastore_postgres.c
+++ b/src/datastore/plugin_datastore_postgres.c
@@ -80,6 +80,7 @@ init_connection (struct Plugin *plugin)
    * we only test equality on it and can cast it to/from uint32_t. For repl, prio, and anonLevel
    * we do math or inequality tests, so we can't handle the entire range of uint32_t.
    * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC.
+   * PostgreSQL also recommends against using WITH OIDS.
    */
   ret =
       PQexec (plugin->dbh,
@@ -114,13 +115,17 @@ init_connection (struct Plugin *plugin)
   if (PQresultStatus (ret) == PGRES_COMMAND_OK)
   {
     if ((GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)")) ||
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)")) ||
         (GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)")) ||
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)")) ||
         (GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_prio ON gn090 (prio)")) ||
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_prio ON gn090 (prio)")) ||
         (GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire)")) ||
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire)")) ||
         (GNUNET_OK !=
          GNUNET_POSTGRES_exec (plugin->dbh,
                                "CREATE INDEX IF NOT EXISTS idx_prio_anon ON gn090 (prio,anonLevel)")) ||
@@ -128,9 +133,11 @@ init_connection (struct Plugin *plugin)
          GNUNET_POSTGRES_exec (plugin->dbh,
                                "CREATE INDEX IF NOT EXISTS idx_prio_hash_anon ON gn090 (prio,hash,anonLevel)")) ||
         (GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)")) ||
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)")) ||
         (GNUNET_OK !=
-         GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_expire_hash ON gn090 (expire,hash)")))
+         GNUNET_POSTGRES_exec (plugin->dbh,
+                               "CREATE INDEX IF NOT EXISTS idx_expire_hash ON gn090 (expire,hash)")))
     {
       PQclear (ret);
       PQfinish (plugin->dbh);
@@ -170,40 +177,18 @@ init_connection (struct Plugin *plugin)
   }
   PQclear (ret);
   if ((GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "getvt",
-                   "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE hash=$1 AND vhash=$2 AND type=$3 "
-                   "ORDER BY oid ASC LIMIT 1 OFFSET $4", 4)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "gett",
-                   "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE hash=$1 AND type=$2 "
-                   "ORDER BY oid ASC LIMIT 1 OFFSET $3", 3)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "getv",
-                   "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE hash=$1 AND vhash=$2 "
-                   "ORDER BY oid ASC LIMIT 1 OFFSET $3", 3)) ||
-      (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "get",
                    "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE hash=$1 " "ORDER BY oid ASC LIMIT 1 OFFSET $2", 2)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "count_getvt",
-                                "SELECT count(*) FROM gn090 WHERE hash=$1 AND vhash=$2 AND type=$3", 3)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "count_gett",
-                                "SELECT count(*) FROM gn090 WHERE hash=$1 AND type=$2", 2)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "count_getv",
-                                "SELECT count(*) FROM gn090 WHERE hash=$1 AND vhash=$2", 2)) ||
-      (GNUNET_OK !=
-       GNUNET_POSTGRES_prepare (plugin->dbh, "count_get",
-                                "SELECT count(*) FROM gn090 WHERE hash=$1", 1)) ||
+                   "WHERE oid >= $1::bigint AND "
+                   "(rvalue >= $2 OR 0 = $3::smallint) AND "
+                   "(hash = $4 OR 0 = $5::smallint) AND "
+                   "(vhash = $6 OR 0 = $7::smallint) AND "
+                   "(type = $8 OR 0 = $9::smallint) "
+                   "ORDER BY oid ASC LIMIT 1", 9)) ||
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "put",
                    "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
-                   "VALUES ($1, $2, $3, $4, $5, RANDOM(), $6, $7, $8)", 9)) ||
+                   "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", 9)) ||
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "update",
                    "UPDATE gn090 SET prio = prio + $1, expire = CASE WHEN expire < $2 THEN $2 ELSE expire END "
@@ -215,8 +200,9 @@ init_connection (struct Plugin *plugin)
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "select_non_anonymous",
                    "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
-                   "WHERE anonLevel = 0 AND type = $1 ORDER BY oid DESC LIMIT 1 OFFSET $2",
-                   1)) ||
+                   "WHERE anonLevel = 0 AND type = $1 AND oid >= $2::bigint "
+                   "ORDER BY oid ASC LIMIT 1",
+                   2)) ||
       (GNUNET_OK !=
        GNUNET_POSTGRES_prepare (plugin->dbh, "select_expiration_order",
                    "(SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 "
@@ -322,6 +308,8 @@ postgres_plugin_put (void *cls,
   struct Plugin *plugin = cls;
   uint32_t utype = type;
   struct GNUNET_HashCode vhash;
+  uint64_t rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                              UINT64_MAX);
   PGresult *ret;
   struct GNUNET_PQ_QueryParam params[] = {
     GNUNET_PQ_query_param_uint32 (&replication),
@@ -329,6 +317,7 @@ postgres_plugin_put (void *cls,
     GNUNET_PQ_query_param_uint32 (&priority),
     GNUNET_PQ_query_param_uint32 (&anonymity),
     GNUNET_PQ_query_param_absolute_time (&expiration),
+    GNUNET_PQ_query_param_uint64 (&rvalue),
     GNUNET_PQ_query_param_auto_from_type (key),
     GNUNET_PQ_query_param_auto_from_type (&vhash),
     GNUNET_PQ_query_param_fixed_size (data, size),
@@ -489,12 +478,11 @@ process_result (struct Plugin *plugin,
 
 
 /**
- * Iterate over the results for a particular key
- * in the datastore.
+ * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure with the 'struct Plugin'
- * @param offset offset of the result (modulo num-results);
- *        specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key maybe NULL (to match all entries)
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
@@ -504,160 +492,52 @@ process_result (struct Plugin *plugin,
  * @param type entries of which type are relevant?
  *     Use 0 for any type.
  * @param proc function to call on the matching value;
- *        will be called once with a NULL if no value matches
- * @param proc_cls closure for iter
+ *        will be called with NULL if nothing matches
+ * @param proc_cls closure for @a proc
  */
 static void
 postgres_plugin_get_key (void *cls,
-                         uint64_t offset,
+                         uint64_t next_uid,
+                         bool random,
                          const struct GNUNET_HashCode *key,
                          const struct GNUNET_HashCode *vhash,
                          enum GNUNET_BLOCK_Type type,
-                         PluginDatumProcessor proc,
+                         PluginDatumProcessor proc,
                          void *proc_cls)
 {
   struct Plugin *plugin = cls;
   uint32_t utype = type;
+  uint16_t use_rvalue = random;
+  uint16_t use_key = NULL != key;
+  uint16_t use_vhash = NULL != vhash;
+  uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type;
+  uint64_t rvalue;
+  struct GNUNET_PQ_QueryParam params[] = {
+    GNUNET_PQ_query_param_uint64 (&next_uid),
+    GNUNET_PQ_query_param_uint64 (&rvalue),
+    GNUNET_PQ_query_param_uint16 (&use_rvalue),
+    GNUNET_PQ_query_param_auto_from_type (key),
+    GNUNET_PQ_query_param_uint16 (&use_key),
+    GNUNET_PQ_query_param_auto_from_type (vhash),
+    GNUNET_PQ_query_param_uint16 (&use_vhash),
+    GNUNET_PQ_query_param_uint32 (&utype),
+    GNUNET_PQ_query_param_uint16 (&use_type),
+    GNUNET_PQ_query_param_end
+  };
   PGresult *ret;
-  uint64_t total;
-  uint64_t limit_off;
 
-  if (0 != type)
+  if (random)
   {
-    if (NULL != vhash)
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_auto_from_type (vhash),
-        GNUNET_PQ_query_param_uint32 (&utype),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "count_getvt",
-                                     params);
-    }
-    else
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_uint32 (&utype),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "count_gett",
-                                     params);
-    }
+    rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX);
+    next_uid = 0;
   }
   else
-  {
-    if (NULL != vhash)
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_auto_from_type (vhash),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "count_getv",
-                                     params);
-    }
-    else
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "count_get",
-                                     params);
-    }
-  }
+    rvalue = 0;
 
-  if (GNUNET_OK !=
-      GNUNET_POSTGRES_check_result (plugin->dbh,
-                                    ret,
-                                    PGRES_TUPLES_OK,
-                                    "PQexecParams",
-                                    "count"))
-  {
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
-          GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  if ( (PQntuples (ret) != 1) ||
-       (PQnfields (ret) != 1) ||
-       (PQgetlength (ret, 0, 0) != sizeof (uint64_t)))
-  {
-    GNUNET_break (0);
-    PQclear (ret);
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
-          GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  total = GNUNET_ntohll (*(const uint64_t *) PQgetvalue (ret, 0, 0));
-  PQclear (ret);
-  if (0 == total)
-  {
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0,
-          GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  limit_off = offset % total;
-
-  if (0 != type)
-  {
-    if (NULL != vhash)
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_auto_from_type (vhash),
-        GNUNET_PQ_query_param_uint32 (&utype),
-        GNUNET_PQ_query_param_uint64 (&limit_off),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "getvt",
-                                     params);
-    }
-    else
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_uint32 (&utype),
-        GNUNET_PQ_query_param_uint64 (&limit_off),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "gett",
-                                     params);
-    }
-  }
-  else
-  {
-    if (NULL != vhash)
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_auto_from_type (vhash),
-        GNUNET_PQ_query_param_uint64 (&limit_off),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "getv",
-                                     params);
-    }
-    else
-    {
-      struct GNUNET_PQ_QueryParam params[] = {
-        GNUNET_PQ_query_param_auto_from_type (key),
-        GNUNET_PQ_query_param_uint64 (&limit_off),
-        GNUNET_PQ_query_param_end
-      };
-      ret = GNUNET_PQ_exec_prepared (plugin->dbh,
-                                     "get",
-                                     params);
-    }
-  }
+  ret = GNUNET_PQ_exec_prepared (plugin->dbh,
+                                 "get",
+                                 params);
   process_result (plugin,
                   proc,
                   proc_cls,
@@ -671,26 +551,25 @@ postgres_plugin_get_key (void *cls,
  * the given iterator for each of them.
  *
  * @param cls our `struct Plugin *`
- * @param offset offset of the result (modulo num-results);
- *        specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
- *        Use 0 for any type.
+ *        Must not be zero (ANY).
  * @param proc function to call on the matching value;
- *        will be called with a NULL if no value matches
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for @a proc
  */
 static void
 postgres_plugin_get_zero_anonymity (void *cls,
-                                    uint64_t offset,
+                                    uint64_t next_uid,
                                     enum GNUNET_BLOCK_Type type,
                                     PluginDatumProcessor proc,
-                                    void *proc_cls)
+                                    void *proc_cls)
 {
   struct Plugin *plugin = cls;
   uint32_t utype = type;
   struct GNUNET_PQ_QueryParam params[] = {
     GNUNET_PQ_query_param_uint32 (&utype),
-    GNUNET_PQ_query_param_uint64 (&offset),
+    GNUNET_PQ_query_param_uint64 (&next_uid),
     GNUNET_PQ_query_param_end
   };
   PGresult *ret;
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c
index 9ab50714f..76f791ad4 100644
--- a/src/datastore/plugin_datastore_sqlite.c
+++ b/src/datastore/plugin_datastore_sqlite.c
@@ -1,6 +1,6 @@
  /*
   * This file is part of GNUnet
-  * Copyright (C) 2009, 2011 GNUnet e.V.
+  * Copyright (C) 2009, 2011, 2017 GNUnet e.V.
   *
   * GNUnet is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License as published
@@ -26,6 +26,7 @@
 
 #include "platform.h"
 #include "gnunet_datastore_plugin.h"
+#include "gnunet_sq_lib.h"
 #include <sqlite3.h>
 
 
@@ -127,6 +128,11 @@ struct Plugin
   sqlite3_stmt *insertContent;
 
   /**
+   * Precompiled SQL for selection
+   */
+  sqlite3_stmt *get;
+
+  /**
    * Should the database be dropped on shutdown?
    */
   int drop_on_shutdown;
@@ -150,11 +156,17 @@ sq_prepare (sqlite3 *dbh,
   char *dummy;
   int result;
 
-  result =
-      sqlite3_prepare_v2 (dbh, zSql, strlen (zSql), ppStmt,
-                          (const char **) &dummy);
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
-                   "Prepared `%s' / %p: %d\n", zSql, *ppStmt, result);
+  result = sqlite3_prepare_v2 (dbh,
+                               zSql,
+                               strlen (zSql),
+                               ppStmt,
+                               (const char **) &dummy);
+  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                   "sqlite",
+                   "Prepared `%s' / %p: %d\n",
+                   zSql,
+                   *ppStmt,
+                   result);
   return result;
 }
 
@@ -310,80 +322,110 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
    * we do math or inequality tests, so we can't handle the entire range of uint32_t.
    * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC.
    */
-  if ((sqlite3_step (stmt) == SQLITE_DONE) &&
-      (sqlite3_exec
-       (plugin->dbh,
-        "CREATE TABLE gn090 (" "  repl INT4 NOT NULL DEFAULT 0,"
-        "  type INT4 NOT NULL DEFAULT 0," "  prio INT4 NOT NULL DEFAULT 0,"
-        "  anonLevel INT4 NOT NULL DEFAULT 0,"
-        "  expire INT8 NOT NULL DEFAULT 0," "  rvalue INT8 NOT NULL,"
-        "  hash TEXT NOT NULL DEFAULT ''," "  vhash TEXT NOT NULL DEFAULT '',"
-        "  value BLOB NOT NULL DEFAULT '')", NULL, NULL, NULL) != SQLITE_OK))
+  if ( (SQLITE_DONE ==
+        sqlite3_step (stmt)) &&
+       (SQLITE_OK !=
+        sqlite3_exec (plugin->dbh,
+                      "CREATE TABLE gn090 ("
+                      "  repl INT4 NOT NULL DEFAULT 0,"
+                      "  type INT4 NOT NULL DEFAULT 0,"
+                      "  prio INT4 NOT NULL DEFAULT 0,"
+                      "  anonLevel INT4 NOT NULL DEFAULT 0,"
+                      "  expire INT8 NOT NULL DEFAULT 0,"
+                      "  rvalue INT8 NOT NULL,"
+                      "  hash TEXT NOT NULL DEFAULT '',"
+                      "  vhash TEXT NOT NULL DEFAULT '',"
+                      "  value BLOB NOT NULL DEFAULT '')",
+                      NULL,
+                      NULL,
+                      NULL)) )
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite3_exec");
     sqlite3_finalize (stmt);
     return GNUNET_SYSERR;
   }
   sqlite3_finalize (stmt);
   create_indices (plugin->dbh);
 
-  if ((sq_prepare
-       (plugin->dbh,
-        "UPDATE gn090 "
-        "SET prio = prio + ?, expire = MAX(expire,?) WHERE _ROWID_ = ?",
-        &plugin->updPrio) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "UPDATE gn090 " "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
-        &plugin->updRepl) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
+  if ( (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "UPDATE gn090 "
+                    "SET prio = prio + ?, expire = MAX(expire,?) WHERE _ROWID_ = ?",
+                    &plugin->updPrio)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "UPDATE gn090 " "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
+                    &plugin->updRepl)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
 #if SQLITE_VERSION_NUMBER >= 3007000
-        "INDEXED BY idx_repl_rvalue "
+                    "INDEXED BY idx_repl_rvalue "
 #endif
-        "WHERE repl=?2 AND " " (rvalue>=?1 OR "
-        "  NOT EXISTS (SELECT 1 FROM gn090 "
+                    "WHERE repl=?2 AND " " (rvalue>=?1 OR "
+                    "  NOT EXISTS (SELECT 1 FROM gn090 "
 #if SQLITE_VERSION_NUMBER >= 3007000
-        "INDEXED BY idx_repl_rvalue "
+                    "INDEXED BY idx_repl_rvalue "
 #endif
-        "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
-        "ORDER BY rvalue ASC LIMIT 1", &plugin->selRepl) != SQLITE_OK) ||
-      (sq_prepare (plugin->dbh, "SELECT MAX(repl) FROM gn090"
+                    "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
+                    "ORDER BY rvalue ASC LIMIT 1",
+                    &plugin->selRepl)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT MAX(repl) FROM gn090"
 #if SQLITE_VERSION_NUMBER >= 3007000
-                   " INDEXED BY idx_repl_rvalue"
+                    " INDEXED BY idx_repl_rvalue"
 #endif
-                   "", &plugin->maxRepl) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
+                    "",
+                    &plugin->maxRepl)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
 #if SQLITE_VERSION_NUMBER >= 3007000
-        "INDEXED BY idx_expire "
+                    "INDEXED BY idx_expire "
 #endif
-        "WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
-        "ORDER BY expire ASC LIMIT 1", &plugin->selExpi) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
+                    "WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
+                    "ORDER BY expire ASC LIMIT 1",
+                    &plugin->selExpi)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 "
 #if SQLITE_VERSION_NUMBER >= 3007000
-        "INDEXED BY idx_anon_type_hash "
+                    "INDEXED BY idx_anon_type_hash "
 #endif
-        "WHERE (anonLevel = 0 AND type=?1) "
-        "ORDER BY hash DESC LIMIT 1 OFFSET ?2",
-        &plugin->selZeroAnon) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
-        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
-        &plugin->insertContent) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh, "DELETE FROM gn090 WHERE _ROWID_ = ?",
-        &plugin->delRow) != SQLITE_OK))
+                    "WHERE _ROWID_ >= ? AND "
+                    "anonLevel = 0 AND "
+                    "type = ? "
+                    "ORDER BY _ROWID_ ASC LIMIT 1",
+                    &plugin->selZeroAnon)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
+                    "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    &plugin->insertContent)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT type, prio, anonLevel, expire, hash, value, _ROWID_ FROM gn090 "
+                    "WHERE _ROWID_ >= ? AND "
+                    "(rvalue >= ? OR 0 = ?) AND "
+                    "(hash = ? OR 0 = ?) AND "
+                    "(vhash = ? OR 0 = ?) AND "
+                    "(type = ? OR 0 = ?) "
+                    "ORDER BY _ROWID_ ASC LIMIT 1",
+                    &plugin->get)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "DELETE FROM gn090 WHERE _ROWID_ = ?",
+                    &plugin->delRow))
+       )
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "precompiling");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "precompiling");
     return GNUNET_SYSERR;
   }
-
   return GNUNET_OK;
 }
 
@@ -398,51 +440,60 @@ static void
 database_shutdown (struct Plugin *plugin)
 {
   int result;
-
 #if SQLITE_VERSION_NUMBER >= 3007000
   sqlite3_stmt *stmt;
 #endif
 
-  if (plugin->delRow != NULL)
+  if (NULL != plugin->delRow)
     sqlite3_finalize (plugin->delRow);
-  if (plugin->updPrio != NULL)
+  if (NULL != plugin->updPrio)
     sqlite3_finalize (plugin->updPrio);
-  if (plugin->updRepl != NULL)
+  if (NULL != plugin->updRepl)
     sqlite3_finalize (plugin->updRepl);
-  if (plugin->selRepl != NULL)
+  if (NULL != plugin->selRepl)
     sqlite3_finalize (plugin->selRepl);
-  if (plugin->maxRepl != NULL)
+  if (NULL != plugin->maxRepl)
     sqlite3_finalize (plugin->maxRepl);
-  if (plugin->selExpi != NULL)
+  if (NULL != plugin->selExpi)
     sqlite3_finalize (plugin->selExpi);
-  if (plugin->selZeroAnon != NULL)
+  if (NULL != plugin->selZeroAnon)
     sqlite3_finalize (plugin->selZeroAnon);
-  if (plugin->insertContent != NULL)
+  if (NULL != plugin->insertContent)
     sqlite3_finalize (plugin->insertContent);
+  if (NULL != plugin->get)
+    sqlite3_finalize (plugin->get);
   result = sqlite3_close (plugin->dbh);
 #if SQLITE_VERSION_NUMBER >= 3007000
   if (result == SQLITE_BUSY)
   {
-    GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
-                     _
-                     ("Tried to close sqlite without finalizing all prepared statements.\n"));
-    stmt = sqlite3_next_stmt (plugin->dbh, NULL);
-    while (stmt != NULL)
+    GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                     "sqlite",
+                     _("Tried to close sqlite without finalizing all prepared statements.\n"));
+    stmt = sqlite3_next_stmt (plugin->dbh,
+                              NULL);
+    while (NULL != stmt)
     {
-      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
-                       "Closing statement %p\n", stmt);
+      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                       "sqlite",
+                       "Closing statement %p\n",
+                       stmt);
       result = sqlite3_finalize (stmt);
       if (result != SQLITE_OK)
-        GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
-                         "Failed to close statement %p: %d\n", stmt, result);
-      stmt = sqlite3_next_stmt (plugin->dbh, NULL);
+        GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                         "sqlite",
+                         "Failed to close statement %p: %d\n",
+                         stmt,
+                         result);
+      stmt = sqlite3_next_stmt (plugin->dbh,
+                                NULL);
     }
     result = sqlite3_close (plugin->dbh);
   }
 #endif
   if (SQLITE_OK != result)
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
-
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite3_close");
   GNUNET_free_non_null (plugin->fn);
 }
 
@@ -456,31 +507,27 @@ database_shutdown (struct Plugin *plugin)
  */
 static int
 delete_by_rowid (struct Plugin *plugin,
-                 unsigned long long rid)
+                 uint64_t rid)
 {
-  if (SQLITE_OK != sqlite3_bind_int64 (plugin->delRow, 1, rid))
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->delRow))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint64 (&rid),
+    GNUNET_SQ_query_param_end
+  };
+
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->delRow,
+                      params))
     return GNUNET_SYSERR;
-  }
   if (SQLITE_DONE != sqlite3_step (plugin->delRow))
   {
     LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-    if (SQLITE_OK != sqlite3_reset (plugin->delRow))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->delRow);
     return GNUNET_SYSERR;
   }
-  if (SQLITE_OK != sqlite3_reset (plugin->delRow))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->delRow);
   return GNUNET_OK;
 }
 
@@ -513,12 +560,25 @@ sqlite_plugin_put (void *cls,
                    PluginPutCont cont,
                    void *cont_cls)
 {
+  uint64_t rvalue;
+  struct GNUNET_HashCode vhash;
+  uint32_t type32 = (uint32_t) type;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint32 (&replication),
+    GNUNET_SQ_query_param_uint32 (&type32),
+    GNUNET_SQ_query_param_uint32 (&priority),
+    GNUNET_SQ_query_param_uint32 (&anonymity),
+    GNUNET_SQ_query_param_absolute_time (&expiration),
+    GNUNET_SQ_query_param_uint64 (&rvalue),
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_auto_from_type (&vhash),
+    GNUNET_SQ_query_param_fixed_size (data, size),
+    GNUNET_SQ_query_param_end
+  };
   struct Plugin *plugin = cls;
   int n;
   int ret;
   sqlite3_stmt *stmt;
-  struct GNUNET_HashCode vhash;
-  uint64_t rvalue;
   char *msg = NULL;
 
   if (size > MAX_ITEM_SIZE)
@@ -537,26 +597,10 @@ sqlite_plugin_put (void *cls,
   GNUNET_CRYPTO_hash (data, size, &vhash);
   stmt = plugin->insertContent;
   rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
-  if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, replication)) ||
-      (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) ||
-      (SQLITE_OK != sqlite3_bind_int (stmt, 3, priority)) ||
-      (SQLITE_OK != sqlite3_bind_int (stmt, 4, anonymity)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 5, expiration.abs_value_us)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 6, rvalue)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_blob (stmt, 7, key, sizeof (struct GNUNET_HashCode),
-                          SQLITE_TRANSIENT)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_blob (stmt, 8, &vhash, sizeof (struct GNUNET_HashCode),
-                          SQLITE_TRANSIENT)) ||
-      (SQLITE_OK != sqlite3_bind_blob (stmt, 9, data, size, SQLITE_TRANSIENT)))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (stmt,
+                      params))
   {
-    LOG_SQLITE_MSG (plugin, &msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
     cont (cont_cls, key, size, GNUNET_SYSERR, msg);
     GNUNET_free_non_null(msg);
     return;
@@ -582,19 +626,16 @@ sqlite_plugin_put (void *cls,
   default:
     LOG_SQLITE_MSG (plugin, &msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                     "sqlite3_step");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
     database_shutdown (plugin);
     database_setup (plugin->env->cfg, plugin);
     cont (cont_cls, key, size, GNUNET_SYSERR, msg);
     GNUNET_free_non_null(msg);
     return;
   }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   cont (cont_cls, key, size, ret, msg);
   GNUNET_free_non_null(msg);
 }
@@ -619,7 +660,7 @@ sqlite_plugin_put (void *cls,
  *     MAX of any existing expiration time and
  *     this value
  * @param cont continuation called with success or failure status
- * @param cons_cls continuation closure
+ * @param cons_cls closure for @a cont
  */
 static void
 sqlite_plugin_update (void *cls,
@@ -630,27 +671,26 @@ sqlite_plugin_update (void *cls,
                       void *cont_cls)
 {
   struct Plugin *plugin = cls;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint32 (&delta),
+    GNUNET_SQ_query_param_absolute_time (&expire),
+    GNUNET_SQ_query_param_uint64 (&uid),
+    GNUNET_SQ_query_param_end
+  };
   int n;
   char *msg = NULL;
 
-  if ((SQLITE_OK != sqlite3_bind_int (plugin->updPrio, 1, delta)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (plugin->updPrio, 2, expire.abs_value_us))
-      || (SQLITE_OK != sqlite3_bind_int64 (plugin->updPrio, 3, uid)))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->updPrio,
+                      params))
   {
-    LOG_SQLITE_MSG (plugin, &msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->updPrio))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
     cont (cont_cls, GNUNET_SYSERR, msg);
     GNUNET_free_non_null(msg);
     return;
   }
   n = sqlite3_step (plugin->updPrio);
-  if (SQLITE_OK != sqlite3_reset (plugin->updPrio))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->updPrio);
   switch (n)
   {
   case SQLITE_DONE:
@@ -691,75 +731,84 @@ execute_get (struct Plugin *plugin,
 {
   int n;
   struct GNUNET_TIME_Absolute expiration;
-  unsigned long long rowid;
-  unsigned int size;
+  uint32_t type;
+  uint32_t priority;
+  uint32_t anonymity;
+  uint64_t rowid;
+  void *value;
+  size_t value_size;
+  struct GNUNET_HashCode key;
   int ret;
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_uint32 (&type),
+    GNUNET_SQ_result_spec_uint32 (&priority),
+    GNUNET_SQ_result_spec_uint32 (&anonymity),
+    GNUNET_SQ_result_spec_absolute_time (&expiration),
+    GNUNET_SQ_result_spec_auto_from_type (&key),
+    GNUNET_SQ_result_spec_variable_size (&value,
+                                         &value_size),
+    GNUNET_SQ_result_spec_uint64 (&rowid),
+    GNUNET_SQ_result_spec_end
+  };
 
   n = sqlite3_step (stmt);
   switch (n)
   {
   case SQLITE_ROW:
-    size = sqlite3_column_bytes (stmt, 5);
-    rowid = sqlite3_column_int64 (stmt, 6);
-    if (sqlite3_column_bytes (stmt, 4) != sizeof (struct GNUNET_HashCode))
+    if (GNUNET_OK !=
+        GNUNET_SQ_extract_result (stmt,
+                                  rs))
     {
-      GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
-                       _("Invalid data in database.  Trying to fix (by deletion).\n"));
-      if (SQLITE_OK != sqlite3_reset (stmt))
-        LOG_SQLITE (plugin,
-                    GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_reset");
-      if ( (GNUNET_OK == delete_by_rowid (plugin, rowid)) &&
-           (NULL != plugin->env->duc) )
-        plugin->env->duc (plugin->env->cls,
-                          -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
+      GNUNET_break (0);
       break;
     }
-    expiration.abs_value_us = sqlite3_column_int64 (stmt, 3);
-    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
+    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                     "sqlite",
                      "Found reply in database with expiration %s\n",
                      GNUNET_STRINGS_absolute_time_to_string (expiration));
-    ret = proc (proc_cls, sqlite3_column_blob (stmt, 4) /* key */ ,
-                size, sqlite3_column_blob (stmt, 5) /* data */ ,
-                sqlite3_column_int (stmt, 0) /* type */ ,
-                sqlite3_column_int (stmt, 1) /* priority */ ,
-                sqlite3_column_int (stmt, 2) /* anonymity */ ,
-                expiration, rowid);
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    ret = proc (proc_cls,
+                &key,
+                value_size,
+                value,
+                type,
+                priority,
+                anonymity,
+                expiration,
+                rowid);
+    GNUNET_SQ_cleanup_result (rs);
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
     if ( (GNUNET_NO == ret) &&
-         (GNUNET_OK == delete_by_rowid (plugin, rowid)) &&
+         (GNUNET_OK == delete_by_rowid (plugin,
+                                        rowid)) &&
          (NULL != plugin->env->duc) )
       plugin->env->duc (plugin->env->cls,
-                        -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
+                        -(value_size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
     return;
   case SQLITE_DONE:
     /* database must be empty */
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
     break;
   case SQLITE_BUSY:
   case SQLITE_ERROR:
   case SQLITE_MISUSE:
   default:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-    if (SQLITE_OK != sqlite3_reset (stmt))
+    if (SQLITE_OK !=
+        sqlite3_reset (stmt))
       LOG_SQLITE (plugin,
                   GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                   "sqlite3_reset");
     GNUNET_break (0);
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     database_shutdown (plugin);
-    database_setup (plugin->env->cfg, plugin);
-    break;
+    database_setup (plugin->env->cfg,
+                    plugin);
+    return;
   }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
 }
 
@@ -769,37 +818,36 @@ execute_get (struct Plugin *plugin,
  * the given processor for the item.
  *
  * @param cls our plugin context
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
- *        Use 0 for any type.
- * @param proc function to call on each matching value;
- *        will be called once with a NULL value at the end
+ *        Must not be zero (ANY).
+ * @param proc function to call on the matching value;
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for @a proc
  */
 static void
-sqlite_plugin_get_zero_anonymity (void *cls, uint64_t offset,
+sqlite_plugin_get_zero_anonymity (void *cls,
+                                  uint64_t next_uid,
                                   enum GNUNET_BLOCK_Type type,
-                                  PluginDatumProcessor proc, void *proc_cls)
+                                  PluginDatumProcessor proc,
+                                  void *proc_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint64 (&next_uid),
+    GNUNET_SQ_query_param_uint32 (&type),
+    GNUNET_SQ_query_param_end
+  };
 
   GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY);
-  stmt = plugin->selZeroAnon;
-  if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) ||
-      (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, offset)))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->selZeroAnon,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
     proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
-  execute_get (plugin, stmt, proc, proc_cls);
+  execute_get (plugin, plugin->selZeroAnon, proc, proc_cls);
 }
 
 
@@ -807,8 +855,9 @@ sqlite_plugin_get_zero_anonymity (void *cls, uint64_t offset,
  * Get results for a particular key in the datastore.
  *
  * @param cls closure
- * @param offset offset (mod count).
- * @param key key to match, never NULL
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
+ * @param key maybe NULL (to match all entries)
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
  *        Note that for DBlocks there is no difference
@@ -822,7 +871,8 @@ sqlite_plugin_get_zero_anonymity (void *cls, uint64_t offset,
  */
 static void
 sqlite_plugin_get_key (void *cls,
-                       uint64_t offset,
+                       uint64_t next_uid,
+                       bool random,
                        const struct GNUNET_HashCode *key,
                        const struct GNUNET_HashCode *vhash,
                        enum GNUNET_BLOCK_Type type,
@@ -830,97 +880,45 @@ sqlite_plugin_get_key (void *cls,
                        void *proc_cls)
 {
   struct Plugin *plugin = cls;
-  int ret;
-  int total;
-  int limit_off;
-  unsigned int sqoff;
-  sqlite3_stmt *stmt;
-  char scratch[256];
-
-  GNUNET_assert (proc != NULL);
-  GNUNET_assert (key != NULL);
-  GNUNET_snprintf (scratch, sizeof (scratch),
-                   "SELECT count(*) FROM gn090 WHERE hash=?%s%s",
-                   vhash == NULL ? "" : " AND vhash=?",
-                   type == 0 ? "" : " AND type=?");
-  if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite_prepare");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  sqoff = 1;
-  ret =
-      sqlite3_bind_blob (stmt, sqoff++, key, sizeof (struct GNUNET_HashCode),
-                         SQLITE_TRANSIENT);
-  if ((vhash != NULL) && (ret == SQLITE_OK))
-    ret =
-        sqlite3_bind_blob (stmt, sqoff++, vhash, sizeof (struct GNUNET_HashCode),
-                           SQLITE_TRANSIENT);
-  if ((type != 0) && (ret == SQLITE_OK))
-    ret = sqlite3_bind_int (stmt, sqoff++, type);
-  if (SQLITE_OK != ret)
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_bind");
-    sqlite3_finalize (stmt);
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  ret = sqlite3_step (stmt);
-  if (ret != SQLITE_ROW)
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite_step");
-    sqlite3_finalize (stmt);
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  total = sqlite3_column_int (stmt, 0);
-  sqlite3_finalize (stmt);
-  if (0 == total)
-  {
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  limit_off = (int) (offset % total);
-  if (limit_off < 0)
-    limit_off += total;
-  GNUNET_snprintf (scratch, sizeof (scratch),
-                   "SELECT type, prio, anonLevel, expire, hash, value, _ROWID_ "
-                   "FROM gn090 WHERE hash=?%s%s "
-                   "ORDER BY _ROWID_ ASC LIMIT 1 OFFSET ?",
-                   vhash == NULL ? "" : " AND vhash=?",
-                   type == 0 ? "" : " AND type=?");
-  if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK)
+  uint64_t rvalue;
+  uint16_t use_rvalue = random;
+  uint32_t type32 = (uint32_t) type;
+  uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type;
+  uint16_t use_key = NULL != key;
+  uint16_t use_vhash = NULL != vhash;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_uint64 (&next_uid),
+    GNUNET_SQ_query_param_uint64 (&rvalue),
+    GNUNET_SQ_query_param_uint16 (&use_rvalue),
+    GNUNET_SQ_query_param_auto_from_type (key),
+    GNUNET_SQ_query_param_uint16 (&use_key),
+    GNUNET_SQ_query_param_auto_from_type (vhash),
+    GNUNET_SQ_query_param_uint16 (&use_vhash),
+    GNUNET_SQ_query_param_uint32 (&type32),
+    GNUNET_SQ_query_param_uint16 (&use_type),
+    GNUNET_SQ_query_param_end
+  };
+
+  if (random)
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite_prepare");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
+    rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX);
+    next_uid = 0;
   }
-  sqoff = 1;
-  ret = sqlite3_bind_blob (stmt, sqoff++, key,
-                           sizeof (struct GNUNET_HashCode),
-                           SQLITE_TRANSIENT);
-  if ((vhash != NULL) && (ret == SQLITE_OK))
-    ret = sqlite3_bind_blob (stmt, sqoff++, vhash,
-                             sizeof (struct GNUNET_HashCode),
-                             SQLITE_TRANSIENT);
-  if ((type != 0) && (ret == SQLITE_OK))
-    ret = sqlite3_bind_int (stmt, sqoff++, type);
-  if (ret == SQLITE_OK)
-    ret = sqlite3_bind_int64 (stmt, sqoff++, limit_off);
-  if (ret != SQLITE_OK)
+  else
+    rvalue = 0;
+
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->get,
+                      params))
   {
-    LOG_SQLITE (plugin,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite_bind");
     proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
-  execute_get (plugin, stmt, proc, proc_cls);
-  sqlite3_finalize (stmt);
+  execute_get (plugin,
+               plugin->get,
+               proc,
+               proc_cls);
 }
 
 
@@ -984,13 +982,17 @@ repl_proc (void *cls,
   struct ReplCtx *rc = cls;
   int ret;
 
+  if (GNUNET_SYSERR == rc->have_uid)
+    rc->have_uid = GNUNET_NO;
   ret = rc->proc (rc->proc_cls,
                   key,
-                  size, data,
+                  size,
+                  data,
                   type,
                   priority,
                   anonymity,
-                  expiration, uid);
+                  expiration,
+                  uid);
   if (NULL != key)
   {
     rc->uid = uid;
@@ -1011,81 +1013,77 @@ repl_proc (void *cls,
  * @param proc_cls closure for @a proc
  */
 static void
-sqlite_plugin_get_replication (void *cls, PluginDatumProcessor proc,
+sqlite_plugin_get_replication (void *cls,
+                               PluginDatumProcessor proc,
                                void *proc_cls)
 {
   struct Plugin *plugin = cls;
   struct ReplCtx rc;
   uint64_t rvalue;
   uint32_t repl;
-  sqlite3_stmt *stmt;
-
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
+  struct GNUNET_SQ_QueryParam params_sel_repl[] = {
+    GNUNET_SQ_query_param_uint64 (&rvalue),
+    GNUNET_SQ_query_param_uint32 (&repl),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_QueryParam params_upd_repl[] = {
+    GNUNET_SQ_query_param_uint64 (&rc.uid),
+    GNUNET_SQ_query_param_end
+  };
+
+  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                   "datastore-sqlite",
                    "Getting random block based on replication order.\n");
-  rc.have_uid = GNUNET_NO;
-  rc.proc = proc;
-  rc.proc_cls = proc_cls;
-  stmt = plugin->maxRepl;
-  if (SQLITE_ROW != sqlite3_step (stmt))
+  if (SQLITE_ROW !=
+      sqlite3_step (plugin->maxRepl))
   {
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->maxRepl);
     /* DB empty */
     proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
-  repl = sqlite3_column_int (stmt, 0);
-  if (SQLITE_OK != sqlite3_reset (stmt))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
-  stmt = plugin->selRepl;
-  rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
-  if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, rvalue))
+  repl = sqlite3_column_int (plugin->maxRepl,
+                             0);
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->maxRepl);
+  rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                     UINT64_MAX);
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->selRepl,
+                      params_sel_repl))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
     proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
-  if (SQLITE_OK != sqlite3_bind_int (stmt, 2, repl))
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
-    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
-    return;
-  }
-  execute_get (plugin, stmt, &repl_proc, &rc);
+  rc.have_uid = GNUNET_SYSERR;
+  rc.proc = proc;
+  rc.proc_cls = proc_cls;
+  execute_get (plugin,
+               plugin->selRepl,
+               &repl_proc,
+               &rc);
   if (GNUNET_YES == rc.have_uid)
   {
-    if (SQLITE_OK != sqlite3_bind_int64 (plugin->updRepl, 1, rc.uid))
+    if (GNUNET_OK !=
+        GNUNET_SQ_bind (plugin->updRepl,
+                        params_upd_repl))
     {
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_bind_XXXX");
-      if (SQLITE_OK != sqlite3_reset (plugin->updRepl))
-        LOG_SQLITE (plugin,
-                    GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_reset");
+      proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
       return;
     }
-    if (SQLITE_DONE != sqlite3_step (plugin->updRepl))
+    if (SQLITE_DONE !=
+        sqlite3_step (plugin->updRepl))
       LOG_SQLITE (plugin,
                   GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                   "sqlite3_step");
-    if (SQLITE_OK != sqlite3_reset (plugin->updRepl))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->updRepl);
+  }
+  if (GNUNET_SYSERR == rc.have_uid)
+  {
+    /* proc was not called at all so far, do it now. */
+    proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
   }
 }
 
@@ -1105,19 +1103,20 @@ sqlite_plugin_get_expiration (void *cls, PluginDatumProcessor proc,
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt;
   struct GNUNET_TIME_Absolute now;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_end
+  };
 
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
+  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                   "sqlite",
                    "Getting random block based on expiration and priority order.\n");
   now = GNUNET_TIME_absolute_get ();
   stmt = plugin->selExpi;
-  if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, now.abs_value_us))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (stmt,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
     proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
     return;
   }
@@ -1138,30 +1137,47 @@ sqlite_plugin_get_keys (void *cls,
                         void *proc_cls)
 {
   struct Plugin *plugin = cls;
-  const struct GNUNET_HashCode *key;
+  struct GNUNET_HashCode key;
+  struct GNUNET_SQ_ResultSpec results[] = {
+    GNUNET_SQ_result_spec_auto_from_type (&key),
+    GNUNET_SQ_result_spec_end
+  };
   sqlite3_stmt *stmt;
   int ret;
 
-  GNUNET_assert (proc != NULL);
-  if (sq_prepare (plugin->dbh, "SELECT hash FROM gn090", &stmt) != SQLITE_OK)
+  GNUNET_assert (NULL != proc);
+  if (SQLITE_OK !=
+      sq_prepare (plugin->dbh,
+                  "SELECT hash FROM gn090",
+                  &stmt))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite_prepare");
-    proc (proc_cls, NULL, 0);
+    proc (proc_cls,
+          NULL,
+          0);
     return;
   }
   while (SQLITE_ROW == (ret = sqlite3_step (stmt)))
   {
-    key = sqlite3_column_blob (stmt, 0);
-    if (sizeof (struct GNUNET_HashCode) == sqlite3_column_bytes (stmt, 0))
-      proc (proc_cls, key, 1);
+    if (GNUNET_OK ==
+        GNUNET_SQ_extract_result (stmt,
+                                  results))
+      proc (proc_cls,
+            &key,
+            1);
     else
       GNUNET_break (0);
   }
   if (SQLITE_DONE != ret)
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite_step");
   sqlite3_finalize (stmt);
-  proc (proc_cls, NULL, 0);
+  proc (proc_cls,
+        NULL,
+        0);
 }
 
 
@@ -1187,7 +1203,8 @@ sqlite_plugin_drop (void *cls)
  * @return the size of the database on disk (estimate)
  */
 static void
-sqlite_plugin_estimate_size (void *cls, unsigned long long *estimate)
+sqlite_plugin_estimate_size (void *cls,
+                             unsigned long long *estimate)
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt;
@@ -1202,30 +1219,45 @@ sqlite_plugin_estimate_size (void *cls, unsigned long long *estimate)
     return;
   if (SQLITE_VERSION_NUMBER < 3006000)
   {
-    GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "datastore-sqlite",
-                     _
-                     ("sqlite version to old to determine size, assuming zero\n"));
+    GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                     "datastore-sqlite",
+                     _("sqlite version to old to determine size, assuming zero\n"));
     *estimate = 0;
     return;
   }
-  CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
+         sqlite3_exec (plugin->dbh,
+                       "VACUUM",
+                       NULL,
+                       NULL,
+                       ENULL));
+  CHECK (SQLITE_OK ==
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA auto_vacuum=INCREMENTAL",
+                       NULL,
                        NULL, ENULL));
-  CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt));
+  CHECK (SQLITE_OK ==
+         sq_prepare (plugin->dbh,
+                     "PRAGMA page_count",
+                     &stmt));
   if (SQLITE_ROW == sqlite3_step (stmt))
-    pages = sqlite3_column_int64 (stmt, 0);
+    pages = sqlite3_column_int64 (stmt,
+                                  0);
   else
     pages = 0;
   sqlite3_finalize (stmt);
-  CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt));
-  CHECK (SQLITE_ROW == sqlite3_step (stmt));
+  CHECK (SQLITE_OK ==
+         sq_prepare (plugin->dbh,
+                     "PRAGMA page_size",
+                     &stmt));
+  CHECK (SQLITE_ROW ==
+         sqlite3_step (stmt));
   page_size = sqlite3_column_int64 (stmt, 0);
   sqlite3_finalize (stmt);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              _
-              ("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
-              (unsigned long long) pages, (unsigned long long) page_size);
+              _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
+              (unsigned long long) pages,
+              (unsigned long long) page_size);
   *estimate = pages * page_size;
 }
 
@@ -1243,9 +1275,11 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
   struct GNUNET_DATASTORE_PluginEnvironment *env = cls;
   struct GNUNET_DATASTORE_PluginFunctions *api;
 
-  if (plugin.env != NULL)
+  if (NULL != plugin.env)
     return NULL;                /* can only initialize once! */
-  memset (&plugin, 0, sizeof (struct Plugin));
+  memset (&plugin,
+          0,
+          sizeof (struct Plugin));
   plugin.env = env;
   if (GNUNET_OK != database_setup (env->cfg, &plugin))
   {
@@ -1263,7 +1297,8 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
   api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity;
   api->get_keys = &sqlite_plugin_get_keys;
   api->drop = &sqlite_plugin_drop;
-  GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "sqlite",
+  GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
+                   "sqlite",
                    _("Sqlite database running\n"));
   return api;
 }
@@ -1282,24 +1317,23 @@ libgnunet_plugin_datastore_sqlite_done (void *cls)
   struct GNUNET_DATASTORE_PluginFunctions *api = cls;
   struct Plugin *plugin = api->cls;
 
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
+  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                   "sqlite",
                    "sqlite plugin is done\n");
   fn = NULL;
   if (plugin->drop_on_shutdown)
     fn = GNUNET_strdup (plugin->fn);
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
-                   "Shutting down database\n");
   database_shutdown (plugin);
   plugin->env = NULL;
   GNUNET_free (api);
-  if (fn != NULL)
+  if (NULL != fn)
   {
     if (0 != UNLINK (fn))
-      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn);
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                "unlink",
+                                fn);
     GNUNET_free (fn);
   }
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
-                   "sqlite plugin is finished\n");
   return NULL;
 }
 
diff --git a/src/datastore/plugin_datastore_template.c b/src/datastore/plugin_datastore_template.c
index a1e03e8ee..187221798 100644
--- a/src/datastore/plugin_datastore_template.c
+++ b/src/datastore/plugin_datastore_template.c
@@ -89,8 +89,8 @@ template_plugin_put (void *cls, const struct GNUNET_HashCode * key, uint32_t siz
  * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key maybe NULL (to match all entries)
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
@@ -104,7 +104,7 @@ template_plugin_put (void *cls, const struct GNUNET_HashCode * key, uint32_t siz
  * @param proc_cls closure for proc
  */
 static void
-template_plugin_get_key (void *cls, uint64_t offset,
+template_plugin_get_key (void *cls, uint64_t next_uid, bool random,
                          const struct GNUNET_HashCode * key,
                          const struct GNUNET_HashCode * vhash,
                          enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc,
@@ -185,16 +185,15 @@ template_plugin_update (void *cls, uint64_t uid, uint32_t delta,
  * Call the given processor on an item with zero anonymity.
  *
  * @param cls our "struct Plugin*"
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
- *        Use 0 for any type.
- * @param proc function to call on each matching value;
- *        will be called  with NULL if no value matches
+ *        Must not be zero (ANY).
+ * @param proc function to call on the matching value;
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for proc
  */
 static void
-template_plugin_get_zero_anonymity (void *cls, uint64_t offset,
+template_plugin_get_zero_anonymity (void *cls, uint64_t next_uid,
                                     enum GNUNET_BLOCK_Type type,
                                     PluginDatumProcessor proc, void *proc_cls)
 {
diff --git a/src/datastore/test_datastore_api.c b/src/datastore/test_datastore_api.c
index a99668240..0da68b266 100644
--- a/src/datastore/test_datastore_api.c
+++ b/src/datastore/test_datastore_api.c
@@ -156,8 +156,6 @@ struct CpsRunContext
   void *data;
   size_t size;
 
-  uint64_t uid;
-  uint64_t offset;
   uint64_t first_uid;
 };
 
@@ -267,7 +265,6 @@ check_value (void *cls,
   GNUNET_assert (priority == get_priority (i));
   GNUNET_assert (anonymity == get_anonymity (i));
   GNUNET_assert (expiration.abs_value_us == get_expiration (i).abs_value_us);
-  crc->offset++;
   if (crc->i == 0)
   {
     crc->phase = RP_DEL;
@@ -343,7 +340,6 @@ check_multiple (void *cls,
   case RP_GET_MULTIPLE:
     crc->phase = RP_GET_MULTIPLE_NEXT;
     crc->first_uid = uid;
-    crc->offset++;
     break;
   case RP_GET_MULTIPLE_NEXT:
     GNUNET_assert (uid != crc->first_uid);
@@ -354,8 +350,6 @@ check_multiple (void *cls,
     crc->phase = RP_ERROR;
     break;
   }
-  if (priority == get_priority (42))
-    crc->uid = uid;
   GNUNET_SCHEDULER_add_now (&run_continuation, crc);
 }
 
@@ -400,7 +394,8 @@ run_continuation (void *cls)
                         sizeof (int),
                         &crc->key);
     GNUNET_DATASTORE_get_key (datastore,
-                              crc->offset,
+                              0,
+                              false,
                               &crc->key,
                               get_type (crc->i),
                               1,
@@ -417,7 +412,8 @@ run_continuation (void *cls)
     GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
     GNUNET_assert (NULL !=
                    GNUNET_DATASTORE_get_key (datastore,
-                                             crc->offset,
+                                             0,
+                                             false,
                                              &crc->key,
                                              get_type (crc->i),
                                              1,
@@ -450,9 +446,15 @@ run_continuation (void *cls)
                 crc->i);
     GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
     GNUNET_assert (NULL !=
-                   GNUNET_DATASTORE_get_key (datastore, crc->offset, &crc->key,
-                                             get_type (crc->i), 1, 1,
-                                             &check_nothing, crc));
+                   GNUNET_DATASTORE_get_key (datastore,
+                                             0,
+                                             false,
+                                             &crc->key,
+                                             get_type (crc->i),
+                                             1,
+                                             1,
+                                             &check_nothing,
+                                             crc));
     break;
   case RP_RESERVE:
     crc->phase = RP_PUT_MULTIPLE;
@@ -483,19 +485,26 @@ run_continuation (void *cls)
   case RP_GET_MULTIPLE:
     GNUNET_assert (NULL !=
                    GNUNET_DATASTORE_get_key (datastore,
-                                             crc->offset,
+                                             0,
+                                             false,
                                              &crc->key,
-                                             get_type (42), 1, 1,
-                                             &check_multiple, crc));
+                                             get_type (42),
+                                             1,
+                                             1,
+                                             &check_multiple,
+                                             crc));
     break;
   case RP_GET_MULTIPLE_NEXT:
     GNUNET_assert (NULL !=
                    GNUNET_DATASTORE_get_key (datastore,
-                                             crc->offset,
+                                             crc->first_uid + 1,
+                                             false,
                                              &crc->key,
                                              get_type (42),
-                                             1, 1,
-                                             &check_multiple, crc));
+                                             1,
+                                             1,
+                                             &check_multiple,
+                                             crc));
     break;
   case RP_DONE:
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
diff --git a/src/datastore/test_datastore_api_management.c b/src/datastore/test_datastore_api_management.c
index 9a3e5446b..de4dc657f 100644
--- a/src/datastore/test_datastore_api_management.c
+++ b/src/datastore/test_datastore_api_management.c
@@ -58,7 +58,6 @@ struct CpsRunContext
   const struct GNUNET_CONFIGURATION_Handle *cfg;
   void *data;
   enum RunPhase phase;
-  uint64_t offset;
 };
 
 
@@ -159,7 +158,6 @@ check_value (void *cls, const struct GNUNET_HashCode * key, size_t size,
   GNUNET_assert (priority == get_priority (i));
   GNUNET_assert (anonymity == get_anonymity (i));
   GNUNET_assert (expiration.abs_value_us == get_expiration (i).abs_value_us);
-  crc->offset++;
   crc->i--;
   if (crc->i == 0)
     crc->phase = RP_DONE;
@@ -221,8 +219,13 @@ run_continuation (void *cls)
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET",
                 crc->i);
     GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
-    GNUNET_DATASTORE_get_key (datastore, crc->offset++, &crc->key,
-                              get_type (crc->i), 1, 1,
+    GNUNET_DATASTORE_get_key (datastore,
+                              0,
+                              false,
+                              &crc->key,
+                              get_type (crc->i),
+                              1,
+                              1,
                               &check_value,
                               crc);
     break;
@@ -230,8 +233,13 @@ run_continuation (void *cls)
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET(f)",
                 crc->i);
     GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key);
-    GNUNET_DATASTORE_get_key (datastore, crc->offset++, &crc->key,
-                              get_type (crc->i), 1, 1,
+    GNUNET_DATASTORE_get_key (datastore,
+                              0,
+                              false,
+                              &crc->key,
+                              get_type (crc->i),
+                              1,
+                              1,
                               &check_nothing,
                               crc);
     break;
diff --git a/src/datastore/test_plugin_datastore.c b/src/datastore/test_plugin_datastore.c
index 9b85d57da..94d93aac6 100644
--- a/src/datastore/test_plugin_datastore.c
+++ b/src/datastore/test_plugin_datastore.c
@@ -64,7 +64,6 @@ struct CpsRunContext
   enum RunPhase phase;
   unsigned int cnt;
   unsigned int i;
-  uint64_t offset;
 };
 
 
@@ -308,7 +307,8 @@ test (void *cls)
                 "Looking for %s\n",
                 GNUNET_h2s (&key));
     crc->api->get_key (crc->api->cls,
-                       crc->offset++,
+                       0,
+                       false,
                        &key,
                        NULL,
                        GNUNET_BLOCK_TYPE_ANY,
diff --git a/src/dht/dht_api.c b/src/dht/dht_api.c
index ee208b50e..42ddc7b60 100644
--- a/src/dht/dht_api.c
+++ b/src/dht/dht_api.c
@@ -319,7 +319,7 @@ send_get_known_results (struct GNUNET_DHT_GetHandle *gh,
   unsigned int max;
   unsigned int transmission_offset;
 
-  max = (GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*msg))
+  max = (GNUNET_MAX_MESSAGE_SIZE - sizeof (*msg))
     / sizeof (struct GNUNET_HashCode);
   transmission_offset = transmission_offset_start;
   while (transmission_offset < gh->seen_results_end)
@@ -704,9 +704,9 @@ check_client_result (void *cls,
     sizeof (struct GNUNET_PeerIdentity) * (get_path_length + put_path_length);
   if ( (msize < meta_length) ||
        (get_path_length >
-        GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
+        GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
        (put_path_length >
-        GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) )
+        GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) )
   {
     GNUNET_break (0);
     return GNUNET_SYSERR;
@@ -998,8 +998,8 @@ GNUNET_DHT_put (struct GNUNET_DHT_Handle *handle,
   struct GNUNET_DHT_PutHandle *ph;
 
   msize = sizeof (struct GNUNET_DHT_ClientPutMessage) + size;
-  if ((msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-      (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE))
+  if ((msize >= GNUNET_MAX_MESSAGE_SIZE) ||
+      (size >= GNUNET_MAX_MESSAGE_SIZE))
   {
     GNUNET_break (0);
     return NULL;
@@ -1090,8 +1090,8 @@ GNUNET_DHT_get_start (struct GNUNET_DHT_Handle *handle,
   size_t msize;
 
   msize = sizeof (struct GNUNET_DHT_ClientGetMessage) + xquery_size;
-  if ((msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-      (xquery_size >= GNUNET_SERVER_MAX_MESSAGE_SIZE))
+  if ((msize >= GNUNET_MAX_MESSAGE_SIZE) ||
+      (xquery_size >= GNUNET_MAX_MESSAGE_SIZE))
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/dht/gnunet-dht-get.c b/src/dht/gnunet-dht-get.c
index ce479dc3e..67f0ce76d 100644
--- a/src/dht/gnunet-dht-get.c
+++ b/src/dht/gnunet-dht-get.c
@@ -50,7 +50,7 @@ static struct GNUNET_TIME_Relative timeout_request = { 60000 };
 /**
  * Be verbose
  */
-static int verbose;
+static unsigned int verbose;
 
 /**
  * Use DHT demultixplex_everywhere
@@ -226,33 +226,6 @@ run (void *cls, char *const *args, const char *cfgfile,
 
 }
 
-
-/**
- * gnunet-dht-get command line options
- */
-static struct GNUNET_GETOPT_CommandLineOption options[] = {
-  {'k', "key", "KEY",
-   gettext_noop ("the query key"),
-   1, &GNUNET_GETOPT_set_string, &query_key},
-  {'r', "replication", "LEVEL",
-   gettext_noop ("how many parallel requests (replicas) to create"),
-   1, &GNUNET_GETOPT_set_uint, &replication},
-  {'t', "type", "TYPE",
-   gettext_noop ("the type of data to look for"),
-   1, &GNUNET_GETOPT_set_uint, &query_type},
-  {'T', "timeout", "TIMEOUT",
-   gettext_noop ("how long to execute this query before giving up?"),
-   1, &GNUNET_GETOPT_set_relative_time, &timeout_request},
-  {'x', "demultiplex", NULL,
-    gettext_noop ("use DHT's demultiplex everywhere option"),
-    0, &GNUNET_GETOPT_set_one, &demultixplex_everywhere},
-  {'V', "verbose", NULL,
-   gettext_noop ("be verbose (print progress information)"),
-   0, &GNUNET_GETOPT_set_one, &verbose},
-  GNUNET_GETOPT_OPTION_END
-};
-
-
 /**
  * Entry point for gnunet-dht-get
  *
@@ -263,6 +236,45 @@ static struct GNUNET_GETOPT_CommandLineOption options[] = {
 int
 main (int argc, char *const *argv)
 {
+
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+  
+    GNUNET_GETOPT_OPTION_STRING ('k',
+                                 "key",
+                                 "KEY",
+                                 gettext_noop ("the query key"),
+                                 &query_key),
+  
+    GNUNET_GETOPT_OPTION_SET_UINT ('r',
+                                   "replication",
+                                   "LEVEL",
+                                   gettext_noop ("how many parallel requests (replicas) to create"),
+                                   &replication),
+  
+  
+    GNUNET_GETOPT_OPTION_SET_UINT ('t',
+                                   "type",
+                                   "TYPE",
+                                   gettext_noop ("the type of data to look for"),
+                                   &query_type),
+  
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('T',
+                                            "timeout",
+                                            "TIMEOUT",
+                                            gettext_noop ("how long to execute this query before giving up?"),
+                                            &timeout_request),
+  
+    GNUNET_GETOPT_OPTION_SET_ONE ('x',
+                                  "demultiplex",
+                                  gettext_noop ("use DHT's demultiplex everywhere option"),
+                                  &demultixplex_everywhere),
+  
+    GNUNET_GETOPT_OPTION_VERBOSE (&verbose),
+    GNUNET_GETOPT_OPTION_END
+  };
+
+
+
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
     return 2;
   return (GNUNET_OK ==
diff --git a/src/dht/gnunet-dht-monitor.c b/src/dht/gnunet-dht-monitor.c
index 7f14255d3..e4e8c46a1 100644
--- a/src/dht/gnunet-dht-monitor.c
+++ b/src/dht/gnunet-dht-monitor.c
@@ -280,27 +280,6 @@ run (void *cls, char *const *args, const char *cfgfile,
                                              NULL);
 }
 
-
-/**
- * gnunet-dht-monitor command line options
- */
-static struct GNUNET_GETOPT_CommandLineOption options[] = {
-  {'k', "key", "KEY",
-   gettext_noop ("the query key"),
-   1, &GNUNET_GETOPT_set_string, &query_key},
-  {'t', "type", "TYPE",
-   gettext_noop ("the type of data to look for"),
-   1, &GNUNET_GETOPT_set_uint, &block_type},
-  {'T', "timeout", "TIMEOUT",
-   gettext_noop ("how long should the monitor command run"),
-   1, &GNUNET_GETOPT_set_relative_time, &timeout_request},
-  {'V', "verbose", NULL,
-   gettext_noop ("be verbose (print progress information)"),
-   0, &GNUNET_GETOPT_set_one, &verbose},
-  GNUNET_GETOPT_OPTION_END
-};
-
-
 /**
  * Entry point for gnunet-dht-monitor
  *
@@ -311,6 +290,35 @@ static struct GNUNET_GETOPT_CommandLineOption options[] = {
 int
 main (int argc, char *const *argv)
 {
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+  
+    GNUNET_GETOPT_OPTION_STRING ('k',
+                                 "key",
+                                 "KEY",
+                                 gettext_noop ("the query key"),
+                                 &query_key),
+  
+    GNUNET_GETOPT_OPTION_SET_UINT ('t',
+                                   "type",
+                                   "TYPE",
+                                   gettext_noop ("the type of data to look for"),
+                                   &block_type),
+  
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('T',
+                                            "timeout",
+                                            "TIMEOUT",
+                                            gettext_noop ("how long should the monitor command run"),
+                                            &timeout_request),
+  
+    GNUNET_GETOPT_OPTION_SET_ONE ('V',
+                                  "verbose",
+                                  gettext_noop ("be verbose (print progress information)"),
+                                  &verbose),
+  
+    GNUNET_GETOPT_OPTION_END
+  };
+
+
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
     return 2;
 
diff --git a/src/dht/gnunet-dht-put.c b/src/dht/gnunet-dht-put.c
index 64764cb65..bf88e5ded 100644
--- a/src/dht/gnunet-dht-put.c
+++ b/src/dht/gnunet-dht-put.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009 GNUnet e.V.
+     Copyright (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -54,7 +54,7 @@ static unsigned int replication = 5;
 /**
  * Be verbose
  */
-static int verbose;
+static unsigned int verbose;
 
 /**
  * Use #GNUNET_DHT_DEMULTIPLEX_EVERYWHERE.
@@ -148,7 +148,6 @@ run (void *cls,
      const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *c)
 {
-  struct GNUNET_TIME_Absolute expiration;
   enum GNUNET_DHT_RouteOption ro;
 
   cfg = c;
@@ -187,44 +186,11 @@ run (void *cls,
                   query_type,
                   strlen (data),
                   data,
-                  expiration,
+                  GNUNET_TIME_relative_to_absolute (expiration),
                   &message_sent_cont,
                   NULL);
 }
 
-
-/**
- * gnunet-dht-put command line options
- */
-static struct GNUNET_GETOPT_CommandLineOption options[] = {
-  {'d', "data", "DATA",
-   gettext_noop ("the data to insert under the key"),
-   1, &GNUNET_GETOPT_set_string, &data},
-  {'e', "expiration", "EXPIRATION",
-   gettext_noop ("how long to store this entry in the dht (in seconds)"),
-   1, &GNUNET_GETOPT_set_relative_time, &expiration},
-  {'k', "key", "KEY",
-   gettext_noop ("the query key"),
-   1, &GNUNET_GETOPT_set_string, &query_key},
-  {'x', "demultiplex", NULL,
-   gettext_noop ("use DHT's demultiplex everywhere option"),
-   0, &GNUNET_GETOPT_set_one, &demultixplex_everywhere},
-  {'r', "replication", "LEVEL",
-   gettext_noop ("how many replicas to create"),
-   1, &GNUNET_GETOPT_set_uint, &replication},
-  {'R', "record", NULL,
-   gettext_noop ("use DHT's record route option"),
-   0, &GNUNET_GETOPT_set_one, &record_route},
-  {'t', "type", "TYPE",
-   gettext_noop ("the type to insert data as"),
-   1, &GNUNET_GETOPT_set_uint, &query_type},
-  {'V', "verbose", NULL,
-   gettext_noop ("be verbose (print progress information)"),
-   0, &GNUNET_GETOPT_set_one, &verbose},
-  GNUNET_GETOPT_OPTION_END
-};
-
-
 /**
  * Entry point for gnunet-dht-put
  *
@@ -235,6 +201,55 @@ static struct GNUNET_GETOPT_CommandLineOption options[] = {
 int
 main (int argc, char *const *argv)
 {
+
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+  
+    GNUNET_GETOPT_OPTION_STRING ('d',
+                                 "data",
+                                 "DATA",
+                                 gettext_noop ("the data to insert under the key"),
+                                 &data),
+  
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('e',
+                                            "expiration",
+                                            "EXPIRATION",
+                                            gettext_noop ("how long to store this entry in the dht (in seconds)"),
+                                            &expiration),
+  
+    GNUNET_GETOPT_OPTION_STRING ('k',
+                                 "key",
+                                 "KEY",
+                                 gettext_noop ("the query key"),
+                                 &query_key),
+  
+    GNUNET_GETOPT_OPTION_SET_ONE ('x',
+                                  "demultiplex",
+                                  gettext_noop ("use DHT's demultiplex everywhere option"),
+                                  &demultixplex_everywhere),
+  
+    GNUNET_GETOPT_OPTION_SET_UINT ('r',
+                                   "replication",
+                                   "LEVEL",
+                                   gettext_noop ("how many replicas to create"),
+                                   &replication),
+  
+    GNUNET_GETOPT_OPTION_SET_ONE ('R',
+                                  "record",
+                                  gettext_noop ("use DHT's record route option"),
+                                  &record_route),
+  
+    GNUNET_GETOPT_OPTION_SET_UINT ('t',
+                                   "type",
+                                   "TYPE",
+                                   gettext_noop ("the type to insert data as"),
+                                   &query_type),
+  
+    GNUNET_GETOPT_OPTION_VERBOSE (&verbose),
+  
+    GNUNET_GETOPT_OPTION_END
+  };
+
+
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv,
                                                  &argc, &argv))
     return 2;
diff --git a/src/dht/gnunet-service-dht_clients.c b/src/dht/gnunet-service-dht_clients.c
index 0f521a401..cb155c484 100644
--- a/src/dht/gnunet-service-dht_clients.c
+++ b/src/dht/gnunet-service-dht_clients.c
@@ -1166,7 +1166,7 @@ GDS_CLIENTS_handle_reply (struct GNUNET_TIME_Absolute expiration,
 
   msize = sizeof (struct GNUNET_DHT_ClientResultMessage) + data_size +
     (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity);
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msize >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
diff --git a/src/dht/gnunet-service-dht_datacache.c b/src/dht/gnunet-service-dht_datacache.c
index fef637cad..36047d561 100644
--- a/src/dht/gnunet-service-dht_datacache.c
+++ b/src/dht/gnunet-service-dht_datacache.c
@@ -72,7 +72,7 @@ GDS_DATACACHE_handle_put (struct GNUNET_TIME_Absolute expiration,
                 _("%s request received, but have no datacache!\n"), "PUT");
     return;
   }
-  if (data_size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (data_size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
diff --git a/src/dht/gnunet-service-dht_neighbours.c b/src/dht/gnunet-service-dht_neighbours.c
index 15071edca..0309bea88 100644
--- a/src/dht/gnunet-service-dht_neighbours.c
+++ b/src/dht/gnunet-service-dht_neighbours.c
@@ -1423,7 +1423,7 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
                                          UINT32_MAX);
   }
   msize = xquery_size + reply_bf_size;
-  if (msize + sizeof (struct PeerGetMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msize + sizeof (struct PeerGetMessage) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     GNUNET_free_non_null (reply_bf);
@@ -1522,12 +1522,12 @@ GDS_NEIGHBOURS_handle_reply (const struct GNUNET_PeerIdentity *target,
 
   msize = data_size + (get_path_length + put_path_length) *
       sizeof (struct GNUNET_PeerIdentity);
-  if ((msize + sizeof (struct PeerResultMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
+  if ((msize + sizeof (struct PeerResultMessage) >= GNUNET_MAX_MESSAGE_SIZE) ||
       (get_path_length >
-       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
+       GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
       (put_path_length >
-       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
-      (data_size > GNUNET_SERVER_MAX_MESSAGE_SIZE))
+       GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
+      (data_size > GNUNET_MAX_MESSAGE_SIZE))
   {
     GNUNET_break (0);
     return;
@@ -1627,7 +1627,7 @@ check_dht_p2p_put (void *cls,
        sizeof (struct PeerPutMessage) +
        putlen * sizeof (struct GNUNET_PeerIdentity)) ||
       (putlen >
-       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
+       GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
@@ -2213,9 +2213,9 @@ check_dht_p2p_result (void *cls,
                                             put_path_length) *
        sizeof (struct GNUNET_PeerIdentity)) ||
       (get_path_length >
-       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
+       GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)) ||
       (put_path_length >
-       GNUNET_SERVER_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
+       GNUNET_MAX_MESSAGE_SIZE / sizeof (struct GNUNET_PeerIdentity)))
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
diff --git a/src/dht/gnunet_dht_profiler.c b/src/dht/gnunet_dht_profiler.c
index 460eaa572..f65141640 100644
--- a/src/dht/gnunet_dht_profiler.c
+++ b/src/dht/gnunet_dht_profiler.c
@@ -1181,31 +1181,54 @@ main (int argc, char *const *argv)
 {
   int rc;
 
-  static struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'n', "peers", "COUNT",
-     gettext_noop ("number of peers to start"),
-     1, &GNUNET_GETOPT_set_uint, &num_peers},
-    {'s', "searches", "COUNT",
-     gettext_noop ("maximum number of times we try to search for successor circle formation (0 for R5N)"),
-     1, &GNUNET_GETOPT_set_uint, &max_searches},
-    {'H', "hosts", "FILENAME",
-     gettext_noop ("name of the file with the login information for the testbed"),
-     1, &GNUNET_GETOPT_set_string, &hosts_file},
-    {'D', "delay", "DELAY",
-     gettext_noop ("delay between rounds for collecting statistics (default: 30 sec)"),
-     1, &GNUNET_GETOPT_set_relative_time, &delay_stats},
-    {'P', "PUT-delay", "DELAY",
-     gettext_noop ("delay to start doing PUTs (default: 1 sec)"),
-     1, &GNUNET_GETOPT_set_relative_time, &delay_put},
-    {'G', "GET-delay", "DELAY",
-     gettext_noop ("delay to start doing GETs (default: 5 min)"),
-     1, &GNUNET_GETOPT_set_relative_time, &delay_get},
-    {'r', "replication", "DEGREE",
-     gettext_noop ("replication degree for DHT PUTs"),
-     1, &GNUNET_GETOPT_set_uint, &replication},
-    {'t', "timeout", "TIMEOUT",
-     gettext_noop ("timeout for DHT PUT and GET requests (default: 1 min)"),
-     1, &GNUNET_GETOPT_set_relative_time, &timeout},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_UINT ('n',
+                                   "peers",
+                                   "COUNT",
+                                   gettext_noop ("number of peers to start"),
+                                   &num_peers),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('s',
+                                   "searches",
+                                   "COUNT",
+                                   gettext_noop ("maximum number of times we try to search for successor circle formation (0 for R5N)"),
+                                   &max_searches),
+
+    GNUNET_GETOPT_OPTION_STRING ('H',
+                                 "hosts",
+                                 "FILENAME",
+                                 gettext_noop ("name of the file with the login information for the testbed"),
+                                 &hosts_file),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('D',
+                                            "delay",
+                                            "DELAY",
+                                            gettext_noop ("delay between rounds for collecting statistics (default: 30 sec)"),
+                                            &delay_stats),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('P',
+                                            "PUT-delay",
+                                            "DELAY",
+                                            gettext_noop ("delay to start doing PUTs (default: 1 sec)"),
+                                            &delay_put),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('G',
+                                            "GET-delay",
+                                            "DELAY",
+                                            gettext_noop ("delay to start doing GETs (default: 5 min)"),
+                                            &delay_get),
+    GNUNET_GETOPT_OPTION_SET_UINT ('r',
+                                   "replication",
+                                   "DEGREE",
+                                   gettext_noop ("replication degree for DHT PUTs"),
+                                   &replication),
+
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('t',
+                                            "timeout",
+                                            "TIMEOUT",
+                                            gettext_noop ("timeout for DHT PUT and GET requests (default: 1 min)"),
+                                            &timeout),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/dns/Makefile.am b/src/dns/Makefile.am
index f74c41fc5..5af228121 100644
--- a/src/dns/Makefile.am
+++ b/src/dns/Makefile.am
@@ -106,6 +106,7 @@ libgnunetdns_la_LDFLAGS = \
 libgnunet_plugin_block_dns_la_SOURCES = \
   plugin_block_dns.c
 libgnunet_plugin_block_dns_la_LIBADD = \
+  $(top_builddir)/src/block/libgnunetblockgroup.la \
   $(top_builddir)/src/util/libgnunetutil.la
 libgnunet_plugin_block_dns_la_LDFLAGS = \
   $(top_builddir)/src/block/$(GN_PLUGIN_LDFLAGS)
diff --git a/src/dns/dns_api.c b/src/dns/dns_api.c
index e7450a1d4..9f0dee9a9 100644
--- a/src/dns/dns_api.c
+++ b/src/dns/dns_api.c
@@ -317,7 +317,7 @@ GNUNET_DNS_request_answer (struct GNUNET_DNS_RequestHandle *rh,
       return;
   }
   if (reply_length + sizeof (struct GNUNET_DNS_Response)
-      >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     GNUNET_free (rh);
diff --git a/src/dns/gnunet-dns-monitor.c b/src/dns/gnunet-dns-monitor.c
index 5e6f90555..2436931fb 100644
--- a/src/dns/gnunet-dns-monitor.c
+++ b/src/dns/gnunet-dns-monitor.c
@@ -52,7 +52,7 @@ static int ret;
 /**
  * Selected level of verbosity.
  */
-static int verbosity;
+static unsigned int verbosity;
 
 
 /**
@@ -346,13 +346,18 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'i', "inbound-only", NULL,
-     gettext_noop ("only monitor DNS queries"),
-     0, &GNUNET_GETOPT_set_one, &inbound_only},
-    {'o', "outbound-only", NULL,
-     gettext_noop ("only monitor DNS replies"),
-     0, &GNUNET_GETOPT_set_one, &outbound_only},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('i',
+                                  "inbound-only",
+                                  gettext_noop ("only monitor DNS queries"),
+                                  &inbound_only),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('o',
+                                  "outbound-only",
+                                  gettext_noop ("only monitor DNS queries"),
+                                  &outbound_only),
+
     GNUNET_GETOPT_OPTION_VERBOSE (&verbosity),
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/dns/gnunet-dns-redirector.c b/src/dns/gnunet-dns-redirector.c
index 89929815a..0469af732 100644
--- a/src/dns/gnunet-dns-redirector.c
+++ b/src/dns/gnunet-dns-redirector.c
@@ -52,7 +52,7 @@ static int ret;
 /**
  * Selected level of verbosity.
  */
-static int verbosity;
+static unsigned int verbosity;
 
 
 /**
@@ -230,13 +230,19 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'4', "ipv4", "IPV4",
-     gettext_noop ("set A records"),
-     1, &GNUNET_GETOPT_set_string, &n4},
-    {'6', "ipv4", "IPV6",
-     gettext_noop ("set AAAA records"),
-     1, &GNUNET_GETOPT_set_string, &n6},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_STRING ('4',
+                                 "ipv4",
+                                 "IPV4",
+                                 gettext_noop ("set A records"),
+                                 &n4),
+
+    GNUNET_GETOPT_OPTION_STRING ('6',
+                                 "ipv4",
+                                 "IPV6",
+                                 gettext_noop ("set AAAA records"),
+                                 &n6),
+
     GNUNET_GETOPT_OPTION_VERBOSE (&verbosity),
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/dns/gnunet-helper-dns.c b/src/dns/gnunet-helper-dns.c
index d083a9df2..5cf1967f5 100644
--- a/src/dns/gnunet-helper-dns.c
+++ b/src/dns/gnunet-helper-dns.c
@@ -79,7 +79,7 @@
 #include "gnunet_protocols.h"
 
 /**
- * Maximum size of a GNUnet message (GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ * Maximum size of a GNUnet message (GNUNET_MAX_MESSAGE_SIZE)
  */
 #define MAX_SIZE 65536
 
diff --git a/src/dns/gnunet-service-dns.c b/src/dns/gnunet-service-dns.c
index 74f595c5e..ffc94afb7 100644
--- a/src/dns/gnunet-service-dns.c
+++ b/src/dns/gnunet-service-dns.c
@@ -347,7 +347,7 @@ request_done (struct RequestRecord *rr)
   }
   reply_len += sizeof (struct GNUNET_TUN_UdpHeader);
   reply_len += rr->payload_length;
-  if (reply_len >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (reply_len >= GNUNET_MAX_MESSAGE_SIZE)
   {
     /* response too big, drop */
     GNUNET_break (0); /* how can this be? */
@@ -481,7 +481,7 @@ send_request_to_client (struct RequestRecord *rr,
   struct GNUNET_MQ_Envelope *env;
   struct GNUNET_DNS_Request *req;
 
-  if (sizeof (struct GNUNET_DNS_Request) + rr->payload_length >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (sizeof (struct GNUNET_DNS_Request) + rr->payload_length >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     cleanup_rr (rr);
@@ -882,11 +882,10 @@ handle_client_response (void *cls,
  * message is received by the tokenizer from the DNS hijack process.
  *
  * @param cls closure
- * @param client identification of the client
  * @param message the actual message, a DNS request we should handle
  */
 static int
-process_helper_messages (void *cls GNUNET_UNUSED, void *client,
+process_helper_messages (void *cls,
                          const struct GNUNET_MessageHeader *message)
 {
   uint16_t msize;
diff --git a/src/dv/dv_api.c b/src/dv/dv_api.c
index 062f9a95f..ab521ec1f 100644
--- a/src/dv/dv_api.c
+++ b/src/dv/dv_api.c
@@ -437,7 +437,7 @@ GNUNET_DV_send (struct GNUNET_DV_ServiceHandle *sh,
   struct ConnectedPeer *peer;
   struct GNUNET_MQ_Envelope *env;
 
-  if (ntohs (msg->size) + sizeof (*sm) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (ntohs (msg->size) + sizeof (*sm) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
diff --git a/src/dv/gnunet-dv.c b/src/dv/gnunet-dv.c
index d0917d363..cb40bb6d2 100644
--- a/src/dv/gnunet-dv.c
+++ b/src/dv/gnunet-dv.c
@@ -34,7 +34,7 @@ static struct GNUNET_DV_ServiceHandle *sh;
 /**
  * Was verbose specified?
  */
-static int verbose;
+static unsigned int verbose;
 
 
 /**
@@ -161,10 +161,10 @@ main (int argc, char *const *argv)
 {
   int res;
 
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'V', "verbose", NULL,
-     gettext_noop ("verbose output"),
-     0, &GNUNET_GETOPT_set_one, &verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_VERBOSE (&verbose),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/dv/gnunet-service-dv.c b/src/dv/gnunet-service-dv.c
index 7d101c8ae..fa1c16be8 100644
--- a/src/dv/gnunet-service-dv.c
+++ b/src/dv/gnunet-service-dv.c
@@ -399,7 +399,7 @@ send_data_to_plugin (const struct GNUNET_MessageHeader *message,
               (unsigned int) distance);
   size = sizeof (struct GNUNET_DV_ReceivedMessage) +
     ntohs (message->size);
-  if (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0); /* too big */
     return;
@@ -537,7 +537,7 @@ forward_payload (struct DirectNeighbor *target,
     return;
   }
   if (sizeof (struct RouteMessage) + ntohs (payload->size)
-      >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -1530,7 +1530,7 @@ listen_set_union (void *cls,
                                         GNUNET_SET_OPERATION_UNION);
   neighbor->set_op = GNUNET_SET_accept (request,
                                         GNUNET_SET_RESULT_ADDED,
-                                        (struct GNUNET_SET_Option[]) { 0 },
+                                        (struct GNUNET_SET_Option[]) {{ 0 }},
                                         &handle_set_union_result,
                                         neighbor);
   neighbor->consensus_insertion_offset = 0;
@@ -1561,7 +1561,7 @@ initiate_set_union (void *cls)
                                          &neighbor->real_session_id,
                                          NULL,
                                          GNUNET_SET_RESULT_ADDED,
-                                         (struct GNUNET_SET_Option[]) { 0 },
+                                         (struct GNUNET_SET_Option[]) {{ 0 }},
                                          &handle_set_union_result,
                                          neighbor);
   neighbor->consensus_insertion_offset = 0;
diff --git a/src/exit/Makefile.am b/src/exit/Makefile.am
index 271b4ebd7..aa1210269 100644
--- a/src/exit/Makefile.am
+++ b/src/exit/Makefile.am
@@ -54,6 +54,6 @@ gnunet_daemon_exit_LDADD = \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(top_builddir)/src/tun/libgnunettun.la \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la \
   $(top_builddir)/src/regex/libgnunetregex.la \
   $(GN_LIBINTL)
diff --git a/src/exit/gnunet-daemon-exit.c b/src/exit/gnunet-daemon-exit.c
index 09576e393..2cd3441a1 100644
--- a/src/exit/gnunet-daemon-exit.c
+++ b/src/exit/gnunet-daemon-exit.c
@@ -979,7 +979,7 @@ send_tcp_packet_via_tun (const struct SocketAddress *destination_address,
   }
   len += sizeof (struct GNUNET_TUN_TcpHeader);
   len += payload_length;
-  if (len >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (len >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -1079,7 +1079,7 @@ send_icmp_packet_via_tun (const struct SocketAddress *destination_address,
   }
   len += sizeof (struct GNUNET_TUN_IcmpHeader);
   len += payload_length;
-  if (len >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (len >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -1358,7 +1358,7 @@ send_udp_packet_via_tun (const struct SocketAddress *destination_address,
   }
   len += sizeof (struct GNUNET_TUN_UdpHeader);
   len += payload_length;
-  if (len >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (len >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -2502,7 +2502,7 @@ store_service (int proto,
               GNUNET_h2s (&cadet_port),
               name,
               (unsigned int) destination_port);
-  service->port = GNUNET_CADET_open_porT (cadet_handle,
+  service->port = GNUNET_CADET_open_port (cadet_handle,
                                           &cadet_port,
                                           &new_service_channel,
                                           service,
@@ -2884,7 +2884,7 @@ tcp_from_helper (const struct GNUNET_TUN_TcpHeader *tcp,
   mtcp->crc = 0;
 
   mlen = sizeof (struct GNUNET_EXIT_TcpDataMessage) + (pktlen - sizeof (struct GNUNET_TUN_TcpHeader));
-  if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -2905,12 +2905,10 @@ tcp_from_helper (const struct GNUNET_TUN_TcpHeader *tcp,
  * Receive packets from the helper-process
  *
  * @param cls unused
- * @param client unsued
  * @param message message received from helper
  */
 static int
 message_token (void *cls GNUNET_UNUSED,
-               void *client GNUNET_UNUSED,
                const struct GNUNET_MessageHeader *message)
 {
   const struct GNUNET_TUN_Layer2PacketHeader *pkt_tun;
@@ -3579,7 +3577,7 @@ advertise_dns_exit ()
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Opening CADET port %s for DNS exit service\n",
               GNUNET_h2s (&port));
-  dns_port = GNUNET_CADET_open_porT (cadet_handle,
+  dns_port = GNUNET_CADET_open_port (cadet_handle,
                                      &port,
                                      &new_channel,
                                      NULL,
@@ -3829,7 +3827,7 @@ run (void *cls,
                                  NULL);
   stats = GNUNET_STATISTICS_create ("exit",
                                     cfg);
-  cadet_handle = GNUNET_CADET_connecT (cfg);
+  cadet_handle = GNUNET_CADET_connect (cfg);
   if (NULL == cadet_handle)
   {
     GNUNET_SCHEDULER_shutdown ();
@@ -3862,7 +3860,7 @@ run (void *cls,
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Opening CADET port %s for IPv4 gateway service\n",
                 GNUNET_h2s (&port));
-    cadet_port4 = GNUNET_CADET_open_porT (cadet_handle,
+    cadet_port4 = GNUNET_CADET_open_port (cadet_handle,
                                           &port,
                                           &new_channel,
                                           NULL,
@@ -3902,7 +3900,7 @@ run (void *cls,
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Opening CADET port %s for IPv6 gateway service\n",
                 GNUNET_h2s (&port));
-    cadet_port6 = GNUNET_CADET_open_porT (cadet_handle,
+    cadet_port6 = GNUNET_CADET_open_port (cadet_handle,
                                           &port,
                                           &new_channel,
                                           NULL,
diff --git a/src/exit/gnunet-helper-exit-windows.c b/src/exit/gnunet-helper-exit-windows.c
index 6be65ccd5..aa7a7a4a3 100644
--- a/src/exit/gnunet-helper-exit-windows.c
+++ b/src/exit/gnunet-helper-exit-windows.c
@@ -77,7 +77,7 @@
 static boolean privilege_testing = FALSE;
 
 /**
- * Maximum size of a GNUnet message (GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ * Maximum size of a GNUnet message (GNUNET_MAX_MESSAGE_SIZE)
  */
 #define MAX_SIZE 65536
 
diff --git a/src/exit/gnunet-helper-exit.c b/src/exit/gnunet-helper-exit.c
index e14c6ca43..3c6f97557 100644
--- a/src/exit/gnunet-helper-exit.c
+++ b/src/exit/gnunet-helper-exit.c
@@ -62,7 +62,7 @@
 #define DEBUG GNUNET_NO
 
 /**
- * Maximum size of a GNUnet message (GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ * Maximum size of a GNUnet message (GNUNET_MAX_MESSAGE_SIZE)
  */
 #define MAX_SIZE 65536
 
diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am
index 4374d45ea..33260a794 100644
--- a/src/fs/Makefile.am
+++ b/src/fs/Makefile.am
@@ -202,7 +202,7 @@ gnunet_service_fs_LDADD =  \
  $(top_builddir)/src/block/libgnunetblock.la \
  $(top_builddir)/src/datastore/libgnunetdatastore.la \
  $(top_builddir)/src/statistics/libgnunetstatistics.la \
- $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+ $(top_builddir)/src/cadet/libgnunetcadet.la \
  $(top_builddir)/src/ats/libgnunetats.la \
  $(top_builddir)/src/core/libgnunetcore.la \
  $(top_builddir)/src/util/libgnunetutil.la \
diff --git a/src/fs/fs_api.h b/src/fs/fs_api.h
index e85de94a7..be22ea73e 100644
--- a/src/fs/fs_api.h
+++ b/src/fs/fs_api.h
@@ -1464,21 +1464,11 @@ struct GNUNET_FS_UnindexContext
   struct GNUNET_CRYPTO_FileHashContext *fhc;
 
   /**
-   * Which values have we seen already?
-   */
-  struct GNUNET_CONTAINER_MultiHashMap *seen_dh;
-
-  /**
    * Overall size of the file.
    */
   uint64_t file_size;
 
   /**
-   * Random offset given to #GNUNET_DATASTORE_get_key.
-   */
-  uint64_t roff;
-
-  /**
    * When did we start?
    */
   struct GNUNET_TIME_Absolute start_time;
diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c
index a18a903be..514eb64b3 100644
--- a/src/fs/fs_directory.c
+++ b/src/fs/fs_directory.c
@@ -170,13 +170,14 @@ find_full_data (void *cls, const char *plugin_name,
  * @param data pointer to the beginning of the directory
  * @param offset offset of data in the directory
  * @param dep function to call on each entry
- * @param dep_cls closure for dep
- * @return GNUNET_OK if this could be a block in a directory,
- *         GNUNET_NO if this could be part of a directory (but not 100% OK)
- *         GNUNET_SYSERR if 'data' does not represent a directory
+ * @param dep_cls closure for @a dep
+ * @return #GNUNET_OK if this could be a block in a directory,
+ *         #GNUNET_NO if this could be part of a directory (but not 100% OK)
+ *         #GNUNET_SYSERR if @a data does not represent a directory
  */
 int
-GNUNET_FS_directory_list_contents (size_t size, const void *data,
+GNUNET_FS_directory_list_contents (size_t size,
+                                   const void *data,
                                    uint64_t offset,
                                    GNUNET_FS_DirectoryEntryProcessor dep,
                                    void *dep_cls)
@@ -194,12 +195,16 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
 
   if ((offset == 0) &&
       ((size < 8 + sizeof (uint32_t)) ||
-       (0 != memcmp (cdata, GNUNET_FS_DIRECTORY_MAGIC, 8))))
+       (0 != memcmp (cdata,
+                     GNUNET_FS_DIRECTORY_MAGIC,
+                     8))))
     return GNUNET_SYSERR;
   pos = offset;
   if (offset == 0)
   {
-    GNUNET_memcpy (&mdSize, &cdata[8], sizeof (uint32_t));
+    GNUNET_memcpy (&mdSize,
+                   &cdata[8],
+                   sizeof (uint32_t));
     mdSize = ntohl (mdSize);
     if (mdSize > size - 8 - sizeof (uint32_t))
     {
@@ -215,7 +220,12 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
       GNUNET_break (0);
       return GNUNET_SYSERR;     /* malformed ! */
     }
-    dep (dep_cls, NULL, NULL, md, 0, NULL);
+    dep (dep_cls,
+         NULL,
+         NULL,
+         md,
+         0,
+         NULL);
     GNUNET_CONTAINER_meta_data_destroy (md);
     pos = 8 + sizeof (uint32_t) + mdSize;
   }
@@ -247,7 +257,7 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
 
     uri = GNUNET_FS_uri_parse (&cdata[pos], &emsg);
     pos = epos + 1;
-    if (uri == NULL)
+    if (NULL == uri)
     {
       GNUNET_free (emsg);
       pos--;                    /* go back to '\0' to force going to next alignment */
@@ -260,7 +270,9 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
       return GNUNET_NO;         /* illegal in directory! */
     }
 
-    GNUNET_memcpy (&mdSize, &cdata[pos], sizeof (uint32_t));
+    GNUNET_memcpy (&mdSize,
+                   &cdata[pos],
+                   sizeof (uint32_t));
     mdSize = ntohl (mdSize);
     pos += sizeof (uint32_t);
     if (pos + mdSize > size)
@@ -269,8 +281,9 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
       return GNUNET_NO;         /* malformed - or partial download */
     }
 
-    md = GNUNET_CONTAINER_meta_data_deserialize (&cdata[pos], mdSize);
-    if (md == NULL)
+    md = GNUNET_CONTAINER_meta_data_deserialize (&cdata[pos],
+                                                 mdSize);
+    if (NULL == md)
     {
       GNUNET_FS_uri_destroy (uri);
       GNUNET_break (0);
@@ -282,10 +295,17 @@ GNUNET_FS_directory_list_contents (size_t size, const void *data,
                                                 EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
     full_data.size = 0;
     full_data.data = NULL;
-    GNUNET_CONTAINER_meta_data_iterate (md, &find_full_data, &full_data);
-    if (dep != NULL)
+    GNUNET_CONTAINER_meta_data_iterate (md,
+                                        &find_full_data,
+                                        &full_data);
+    if (NULL != dep)
     {
-      dep (dep_cls, filename, uri, md, full_data.size, full_data.data);
+      dep (dep_cls,
+           filename,
+           uri,
+           md,
+           full_data.size,
+           full_data.data);
     }
     GNUNET_free_non_null (full_data.data);
     GNUNET_free_non_null (filename);
@@ -548,11 +568,12 @@ block_align (size_t start, unsigned int count, const size_t * sizes,
  * @param bld directory to finish
  * @param rsize set to the number of bytes needed
  * @param rdata set to the encoded directory
- * @return GNUNET_OK on success
+ * @return #GNUNET_OK on success
  */
 int
 GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
-                                    size_t * rsize, void **rdata)
+                                    size_t * rsize,
+                                    void **rdata)
 {
   char *data;
   char *sptr;
@@ -575,9 +596,12 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
   bes = NULL;
   if (0 < bld->count)
   {
-    sizes = GNUNET_malloc (bld->count * sizeof (size_t));
-    perm = GNUNET_malloc (bld->count * sizeof (unsigned int));
-    bes = GNUNET_malloc (bld->count * sizeof (struct BuilderEntry *));
+    sizes = GNUNET_new_array (bld->count,
+                              size_t);
+    perm = GNUNET_new_array (bld->count,
+                             unsigned int);
+    bes = GNUNET_new_array (bld->count,
+                            struct BuilderEntry *);
     pos = bld->head;
     for (i = 0; i < bld->count; i++)
     {
@@ -599,7 +623,8 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
   data = GNUNET_malloc_large (size);
   if (data == NULL)
   {
-    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "malloc");
+    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
+                         "malloc");
     *rsize = 0;
     *rdata = NULL;
     GNUNET_free_non_null (sizes);
@@ -608,17 +633,22 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
     return GNUNET_SYSERR;
   }
   *rdata = data;
-  GNUNET_memcpy (data, GNUNET_DIRECTORY_MAGIC, strlen (GNUNET_DIRECTORY_MAGIC));
+  GNUNET_memcpy (data,
+                 GNUNET_DIRECTORY_MAGIC,
+                 strlen (GNUNET_DIRECTORY_MAGIC));
   off = strlen (GNUNET_DIRECTORY_MAGIC);
 
   sptr = &data[off + sizeof (uint32_t)];
   ret =
-      GNUNET_CONTAINER_meta_data_serialize (bld->meta, &sptr,
+      GNUNET_CONTAINER_meta_data_serialize (bld->meta,
+                                            &sptr,
                                             size - off - sizeof (uint32_t),
                                             GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
   GNUNET_assert (ret != -1);
   big = htonl (ret);
-  GNUNET_memcpy (&data[off], &big, sizeof (uint32_t));
+  GNUNET_memcpy (&data[off],
+                 &big,
+                 sizeof (uint32_t));
   off += sizeof (uint32_t) + ret;
   for (j = 0; j < bld->count; j++)
   {
diff --git a/src/fs/fs_dirmetascan.c b/src/fs/fs_dirmetascan.c
index 2f79c7c05..7b9f178fd 100644
--- a/src/fs/fs_dirmetascan.c
+++ b/src/fs/fs_dirmetascan.c
@@ -245,12 +245,10 @@ finish_scan (void *cls)
  * Calls the scanner progress handler.
  *
  * @param cls the closure (directory scanner object)
- * @param client always NULL
  * @param msg message from the helper process
  */
 static int
 process_helper_msgs (void *cls,
-                     void *client,
                      const struct GNUNET_MessageHeader *msg)
 {
   struct GNUNET_FS_DirScanner *ds = cls;
diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c
index 0789162bf..ce852f2d0 100644
--- a/src/fs/fs_download.c
+++ b/src/fs/fs_download.c
@@ -37,14 +37,14 @@ static int
 is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
 {
   return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
-      ((GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
-       ((NULL == dc->meta) &&
-        ((NULL == dc->filename) ||
-         ((strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
-          (NULL !=
-           strstr (dc->filename + strlen (dc->filename) -
-                   strlen (GNUNET_FS_DIRECTORY_EXT),
-                   GNUNET_FS_DIRECTORY_EXT))))));
+      ( (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
+        ( (NULL == dc->meta) &&
+          ( (NULL == dc->filename) ||
+            ( (strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
+              (NULL !=
+               strstr (dc->filename + strlen (dc->filename) -
+                       strlen (GNUNET_FS_DIRECTORY_EXT),
+                       GNUNET_FS_DIRECTORY_EXT)) ) ) ) );
 }
 
 
@@ -278,10 +278,12 @@ try_reconnect (struct GNUNET_FS_DownloadContext *dc);
  * @param data contents of the file (or NULL if they were not inlined)
  */
 static void
-trigger_recursive_download (void *cls, const char *filename,
+trigger_recursive_download (void *cls,
+                            const char *filename,
                             const struct GNUNET_FS_Uri *uri,
                             const struct GNUNET_CONTAINER_MetaData *meta,
-                            size_t length, const void *data);
+                            size_t length,
+                            const void *data);
 
 
 /**
@@ -304,24 +306,28 @@ full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
   if (size64 != (uint64_t) size)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                _
-                ("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
+                _("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
     return;
   }
   if (NULL != dc->filename)
   {
-    h = GNUNET_DISK_file_open (dc->filename, GNUNET_DISK_OPEN_READ,
+    h = GNUNET_DISK_file_open (dc->filename,
+                               GNUNET_DISK_OPEN_READ,
                                GNUNET_DISK_PERM_NONE);
   }
   else
   {
     GNUNET_assert (NULL != dc->temp_filename);
-    h = GNUNET_DISK_file_open (dc->temp_filename, GNUNET_DISK_OPEN_READ,
+    h = GNUNET_DISK_file_open (dc->temp_filename,
+                               GNUNET_DISK_OPEN_READ,
                                GNUNET_DISK_PERM_NONE);
   }
   if (NULL == h)
     return;                     /* oops */
-  data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
+  data = GNUNET_DISK_file_map (h,
+                               &m,
+                               GNUNET_DISK_MAP_TYPE_READ,
+                               size);
   if (NULL == data)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -329,15 +335,25 @@ full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
   }
   else
   {
-    GNUNET_FS_directory_list_contents (size, data, 0,
-                                       &trigger_recursive_download, dc);
+    if (GNUNET_OK !=
+        GNUNET_FS_directory_list_contents (size,
+                                           data,
+                                           0,
+                                           &trigger_recursive_download,
+                                           dc))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _("Failed to access full directroy contents of `%s' for recursive download\n"),
+                  dc->filename);
+    }
     GNUNET_DISK_file_unmap (m);
   }
   GNUNET_DISK_file_close (h);
   if (NULL == dc->filename)
   {
     if (0 != UNLINK (dc->temp_filename))
-      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                                "unlink",
                                 dc->temp_filename);
     GNUNET_free (dc->temp_filename);
     dc->temp_filename = NULL;
@@ -362,14 +378,16 @@ check_completed (struct GNUNET_FS_DownloadContext *dc)
   struct GNUNET_FS_DownloadContext *pos;
 
   /* first, check if we need to download children */
-  if ((NULL == dc->child_head) && (is_recursive_download (dc)))
+  if (is_recursive_download (dc))
     full_recursive_download (dc);
   /* then, check if children are done already */
   for (pos = dc->child_head; NULL != pos; pos = pos->next)
   {
-    if ((pos->emsg == NULL) && (pos->completed < pos->length))
+    if ( (NULL == pos->emsg) &&
+         (pos->completed < pos->length) )
       return;                   /* not done yet */
-    if ((pos->child_head != NULL) && (pos->has_finished != GNUNET_YES))
+    if ( (NULL != pos->child_head) &&
+         (pos->has_finished != GNUNET_YES) )
       return;                   /* not transitively done yet */
   }
   /* All of our children are done, so mark this download done */
@@ -471,7 +489,11 @@ try_match_block (struct GNUNET_FS_DownloadContext *dc,
   }
   GNUNET_CRYPTO_hash (&data[dr->offset], dlen, &in_chk.key);
   GNUNET_CRYPTO_hash_to_aes_key (&in_chk.key, &sk, &iv);
-  if (-1 == GNUNET_CRYPTO_symmetric_encrypt (&data[dr->offset], dlen, &sk, &iv, enc))
+  if (-1 == GNUNET_CRYPTO_symmetric_encrypt (&data[dr->offset],
+                                             dlen,
+                                             &sk,
+                                             &iv,
+                                             enc))
   {
     GNUNET_break (0);
     return;
@@ -484,7 +506,9 @@ try_match_block (struct GNUNET_FS_DownloadContext *dc,
     dr->state = BRS_RECONSTRUCT_META_UP;
     break;
   case BRS_CHK_SET:
-    if (0 != memcmp (&in_chk, &dr->chk, sizeof (struct ContentHashKey)))
+    if (0 != memcmp (&in_chk,
+                     &dr->chk,
+                     sizeof (struct ContentHashKey)))
     {
       /* other peer provided bogus meta data */
       GNUNET_break_op (0);
@@ -591,7 +615,10 @@ match_full_data (void *cls, const char *plugin_name,
     GNUNET_break_op (0);
     return 1;                   /* bogus meta data */
   }
-  try_match_block (dc, dc->top_request, data, data_len);
+  try_match_block (dc,
+                   dc->top_request,
+                   data,
+                   data_len);
   return 1;
 }
 
@@ -820,10 +847,12 @@ schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
  * @param data contents of the file (or NULL if they were not inlined)
  */
 static void
-trigger_recursive_download (void *cls, const char *filename,
+trigger_recursive_download (void *cls,
+                            const char *filename,
                             const struct GNUNET_FS_Uri *uri,
                             const struct GNUNET_CONTAINER_MetaData *meta,
-                            size_t length, const void *data)
+                            size_t length,
+                            const void *data)
 {
   struct GNUNET_FS_DownloadContext *dc = cls;
   struct GNUNET_FS_DownloadContext *cpos;
@@ -936,9 +965,17 @@ trigger_recursive_download (void *cls, const char *filename,
               (unsigned long long) GNUNET_FS_uri_chk_get_file_size (uri),
               (unsigned int)
               GNUNET_CONTAINER_meta_data_get_serialized_size (meta));
-  GNUNET_FS_download_start (dc->h, uri, meta, full_name, temp_name, 0,
+  GNUNET_FS_download_start (dc->h,
+                            uri,
+                            meta,
+                            full_name,
+                            temp_name,
+                            0,
                             GNUNET_FS_uri_chk_get_file_size (uri),
-                            dc->anonymity, dc->options, NULL, dc);
+                            dc->anonymity,
+                            dc->options,
+                            NULL,
+                            dc);
   GNUNET_free_non_null (full_name);
   GNUNET_free_non_null (temp_name);
   GNUNET_free_non_null (fn);
@@ -953,11 +990,9 @@ trigger_recursive_download (void *cls, const char *filename,
 void
 GNUNET_FS_free_download_request_ (struct DownloadRequest *dr)
 {
-  unsigned int i;
-
   if (NULL == dr)
     return;
-  for (i = 0; i < dr->num_children; i++)
+  for (unsigned int i = 0; i < dr->num_children; i++)
     GNUNET_FS_free_download_request_ (dr->children[i]);
   GNUNET_free_non_null (dr->children);
   GNUNET_free (dr);
@@ -1509,13 +1544,17 @@ create_download_request (struct DownloadRequest *parent,
   GNUNET_assert (dr->num_children > 0);
 
   dr->children =
-    GNUNET_malloc (dr->num_children * sizeof (struct DownloadRequest *));
+    GNUNET_new_array (dr->num_children,
+                      struct DownloadRequest *);
   for (i = 0; i < dr->num_children; i++)
   {
     dr->children[i] =
-      create_download_request (dr, i + head_skip, depth - 1,
+      create_download_request (dr,
+                               i + head_skip,
+                               depth - 1,
                                dr_offset + (i + head_skip) * child_block_size,
-                               file_start_offset, desired_length);
+                               file_start_offset,
+                               desired_length);
   }
   return dr;
 }
diff --git a/src/fs/fs_getopt.c b/src/fs/fs_getopt.c
index f78e311d3..bfe45957e 100644
--- a/src/fs/fs_getopt.c
+++ b/src/fs/fs_getopt.c
@@ -25,6 +25,7 @@
  */
 #include "platform.h"
 #include "gnunet_fs_service.h"
+#include "gnunet_getopt_lib.h"
 #include "fs_api.h"
 
 /* ******************** command-line option parsing API ******************** */
@@ -41,10 +42,10 @@
  * @param value command line argument given
  * @return GNUNET_OK on success
  */
-int
-GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext
-                               *ctx, void *scls, const char *option,
-                               const char *value)
+static int
+getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext
+                     *ctx, void *scls, const char *option,
+                      const char *value)
 {
   struct GNUNET_FS_Uri **uri = scls;
   struct GNUNET_FS_Uri *u = *uri;
@@ -107,6 +108,34 @@ GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext
   return GNUNET_OK;
 }
 
+/**
+ * Allow user to specify keywords.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] topKeywords set to the desired value
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_FS_GETOPT_KEYWORDS (char shortName,
+                           const char *name,
+                           const char *argumentHelp,
+                           const char *description,
+                           struct GNUNET_FS_Uri **topKeywords)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &getopt_set_keywords,
+    .scls = (void *) topKeywords  
+  };
+
+  return clo;
+}
 
 /**
  * Command-line option parser function that allows the user to specify
@@ -120,11 +149,11 @@ GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext
  * @param value command line argument given
  * @return #GNUNET_OK on success
  */
-int
-GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                               void *scls,
-                               const char *option,
-                               const char *value)
+static int
+getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+                     void *scls,
+                     const char *option,
+                     const char *value)
 {
   struct GNUNET_CONTAINER_MetaData **mm = scls;
 #if HAVE_EXTRACTOR_H && HAVE_LIBEXTRACTOR
@@ -200,4 +229,36 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext
   return GNUNET_OK;
 }
 
+/**
+ * Allow user to specify metadata.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] metadata set to the desired value
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_FS_GETOPT_METADATA (char shortName,
+                           const char *name,
+                           const char *argumentHelp,
+                           const char *description,
+                           struct GNUNET_CONTAINER_MetaData **meta)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &getopt_set_metadata,
+    .scls = (void *) meta
+  };
+
+  return clo;
+}
+
+
+
+
 /* end of fs_getopt.c */
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c
index 86a58a58b..759467ba4 100644
--- a/src/fs/fs_publish.c
+++ b/src/fs/fs_publish.c
@@ -837,7 +837,7 @@ hash_for_index_cb (void *cls,
   GNUNET_assert (fn != NULL);
   slen = strlen (fn) + 1;
   if (slen >=
-      GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct IndexStartMessage))
+      GNUNET_MAX_MESSAGE_SIZE - sizeof (struct IndexStartMessage))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _
@@ -1226,7 +1226,7 @@ fip_signal_start (void *cls,
   {
     kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri);
     pc->reserve_entries += kc;
-    pc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc;
+    pc->reserve_space += GNUNET_MAX_MESSAGE_SIZE * kc;
   }
   pi.status = GNUNET_FS_STATUS_PUBLISH_START;
   *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0);
diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c
index 3cbf2afef..8c6f5edcf 100644
--- a/src/fs/fs_search.c
+++ b/src/fs/fs_search.c
@@ -1088,15 +1088,17 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
   unsigned int left;
   unsigned int todo;
   unsigned int fit;
-  int first_call;
   unsigned int search_request_map_offset;
   unsigned int keyword_offset;
+  int first_call;
 
   memset (&mbc, 0, sizeof (mbc));
   mbc.sc = sc;
   if (GNUNET_FS_uri_test_ksk (sc->uri))
   {
-    mbc.put_cnt = 0;
+    /* This will calculate the result set size ONLY for
+       "keyword_offset == 0", so we will have to recalculate
+       it for the other keywords later! */
     GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
                                            &find_result_set,
                                            &mbc);
@@ -1109,7 +1111,6 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
   }
   search_request_map_offset = 0;
   keyword_offset = 0;
-
   first_call = GNUNET_YES;
   while ( (0 != (left =
                  (total_seen_results - search_request_map_offset))) ||
@@ -1120,7 +1121,7 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
     if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY))
       options |= SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY;
 
-    fit = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - sizeof (*sm)) / sizeof (struct GNUNET_HashCode);
+    fit = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof (*sm)) / sizeof (struct GNUNET_HashCode);
     todo = GNUNET_MIN (fit,
                        left);
     env = GNUNET_MQ_msg_extra (sm,
@@ -1128,6 +1129,11 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
                                GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
     mbc.skip_cnt = search_request_map_offset;
     mbc.xoff = (struct GNUNET_HashCode *) &sm[1];
+    sm->type = htonl (GNUNET_BLOCK_TYPE_FS_UBLOCK);
+    sm->anonymity_level = htonl (sc->anonymity);
+    memset (&sm->target,
+            0,
+            sizeof (struct GNUNET_PeerIdentity));
 
     if (GNUNET_FS_uri_test_ksk (sc->uri))
     {
@@ -1135,17 +1141,12 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
       /* calculate how many results we can send in this message */
       mbc.put_cnt = todo;
       /* now build message */
-      sm->type = htonl (GNUNET_BLOCK_TYPE_FS_UBLOCK);
-      sm->anonymity_level = htonl (sc->anonymity);
-      memset (&sm->target,
-              0,
-              sizeof (struct GNUNET_PeerIdentity));
       sm->query = sc->requests[keyword_offset].uquery;
       GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
                                              &build_result_set,
                                              &mbc);
       search_request_map_offset += todo;
-      GNUNET_assert (0 == mbc.put_cnt); /* #4608 reports this fails? */
+      GNUNET_assert (0 == mbc.put_cnt);
       GNUNET_assert (total_seen_results >= search_request_map_offset);
       if (total_seen_results != search_request_map_offset)
       {
@@ -1156,11 +1157,17 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
       {
         sm->options = htonl (options);
         keyword_offset++;
-        search_request_map_offset = 0;
         if (sc->uri->data.ksk.keywordCount != keyword_offset)
         {
           /* more keywords => more requesting to be done... */
           first_call = GNUNET_YES;
+          search_request_map_offset = 0;
+          mbc.put_cnt = 0;
+          mbc.keyword_offset = keyword_offset;
+          GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
+                                                 &find_result_set,
+                                                 &mbc);
+          total_seen_results = mbc.put_cnt;
         }
       }
     }
@@ -1168,11 +1175,6 @@ schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc)
     {
       GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri));
 
-      sm->type = htonl (GNUNET_BLOCK_TYPE_FS_UBLOCK);
-      sm->anonymity_level = htonl (sc->anonymity);
-      memset (&sm->target,
-              0,
-              sizeof (struct GNUNET_PeerIdentity));
       GNUNET_CRYPTO_ecdsa_public_key_derive (&sc->uri->data.sks.ns,
                                              sc->uri->data.sks.identifier,
                                              "fs-ublock",
@@ -1383,9 +1385,10 @@ GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc)
     GNUNET_assert (0 != sc->uri->data.ksk.keywordCount);
     anon = GNUNET_CRYPTO_ecdsa_key_get_anonymous ();
     GNUNET_CRYPTO_ecdsa_key_get_public (anon, &anon_pub);
-    sc->requests =
-        GNUNET_malloc (sizeof (struct SearchRequestEntry) *
-                       sc->uri->data.ksk.keywordCount);
+    sc->requests
+      = GNUNET_new_array (sc->uri->data.ksk.keywordCount,
+                          struct SearchRequestEntry);
+
     for (i = 0; i < sc->uri->data.ksk.keywordCount; i++)
     {
       keyword = &sc->uri->data.ksk.keywords[i][1];
@@ -1666,8 +1669,8 @@ search_result_stop (void *cls,
   if (NULL != sr->download)
   {
     sr->download->search = NULL;
-    sr->download->top =
-        GNUNET_FS_make_top (sr->download->h,
+    sr->download->top
+      = GNUNET_FS_make_top (sr->download->h,
                             &GNUNET_FS_download_signal_suspend_,
                             sr->download);
     if (NULL != sr->download->serialization)
@@ -1679,7 +1682,8 @@ search_result_stop (void *cls,
       sr->download->serialization = NULL;
     }
     pi.status = GNUNET_FS_STATUS_DOWNLOAD_LOST_PARENT;
-    GNUNET_FS_download_make_status_ (&pi, sr->download);
+    GNUNET_FS_download_make_status_ (&pi,
+                                     sr->download);
     GNUNET_FS_download_sync_ (sr->download);
     sr->download = NULL;
   }
@@ -1745,25 +1749,28 @@ GNUNET_FS_search_stop (struct GNUNET_FS_SearchContext *sc)
   if (NULL != sc->top)
     GNUNET_FS_end_top (sc->h, sc->top);
   GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map,
-                                         &search_result_stop, sc);
+                                         &search_result_stop,
+                                         sc);
   if (NULL != sc->psearch_result)
     sc->psearch_result->update_search = NULL;
   if (NULL != sc->serialization)
   {
     GNUNET_FS_remove_sync_file_ (sc->h,
-                                 (sc->psearch_result !=
-                                  NULL) ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH :
-                                 GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
+                                 (NULL != sc->psearch_result)
+                                 ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH
+                                 : GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
                                  sc->serialization);
     GNUNET_FS_remove_sync_dir_ (sc->h,
-                                (sc->psearch_result !=
-                                 NULL) ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH :
-                                GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
+                                (NULL != sc->psearch_result)
+                                ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH
+                                : GNUNET_FS_SYNC_PATH_MASTER_SEARCH,
                                 sc->serialization);
     GNUNET_free (sc->serialization);
   }
   pi.status = GNUNET_FS_STATUS_SEARCH_STOPPED;
-  sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc->h, sc);
+  sc->client_info = GNUNET_FS_search_make_status_ (&pi,
+                                                   sc->h,
+                                                   sc);
   GNUNET_break (NULL == sc->client_info);
   if (NULL != sc->task)
   {
diff --git a/src/fs/fs_tree.c b/src/fs/fs_tree.c
index b3c632203..e57e4e494 100644
--- a/src/fs/fs_tree.c
+++ b/src/fs/fs_tree.c
@@ -285,9 +285,9 @@ GNUNET_FS_tree_encoder_create (struct GNUNET_FS_Handle *h, uint64_t size,
   te->progress = progress;
   te->cont = cont;
   te->chk_tree_depth = GNUNET_FS_compute_depth (size);
-  te->chk_tree =
-      GNUNET_malloc (te->chk_tree_depth * CHK_PER_INODE *
-                     sizeof (struct ContentHashKey));
+  te->chk_tree
+    = GNUNET_new_array (te->chk_tree_depth * CHK_PER_INODE,
+                        struct ContentHashKey);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Created tree encoder for file with %llu bytes and depth %u\n",
               (unsigned long long) size,
diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c
index a672b84d5..e1c7ea535 100644
--- a/src/fs/fs_unindex.c
+++ b/src/fs/fs_unindex.c
@@ -312,8 +312,6 @@ unindex_finish (struct GNUNET_FS_UnindexContext *uc)
   uc->fh = NULL;
   GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO);
   uc->dsh = NULL;
-  GNUNET_CONTAINER_multihashmap_destroy (uc->seen_dh);
-  uc->seen_dh = NULL;
   uc->state = UNINDEX_STATE_FS_NOTIFY;
   GNUNET_FS_unindex_sync_ (uc);
   uc->mq = GNUNET_CLIENT_connect (uc->h->cfg,
@@ -444,7 +442,6 @@ continue_after_remove (void *cls,
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _("Failed to remove UBlock: %s\n"),
                 msg);
-  GNUNET_CONTAINER_multihashmap_clear (uc->seen_dh);
   uc->ksk_offset++;
   GNUNET_FS_unindex_do_remove_kblocks_ (uc);
 }
@@ -454,7 +451,8 @@ continue_after_remove (void *cls,
  * Function called from datastore with result from us looking for
  * a UBlock.  There are four cases:
  * 1) no result, means we move on to the next keyword
- * 2) UID is the same as the first UID, means we move on to next keyword
+ * 2) data hash is the same as an already seen data hash, means we move on to
+ *    next keyword
  * 3) UBlock for a different CHK, means we keep looking for more
  * 4) UBlock is for our CHK, means we remove the block and then move
  *           on to the next keyword
@@ -485,34 +483,15 @@ process_kblock_for_unindex (void *cls,
   const struct UBlock *ub;
   struct GNUNET_FS_Uri *chk_uri;
   struct GNUNET_HashCode query;
-  struct GNUNET_HashCode dh;
 
   uc->dqe = NULL;
   if (NULL == data)
   {
     /* no result */
-    GNUNET_CONTAINER_multihashmap_clear (uc->seen_dh);
     uc->ksk_offset++;
     GNUNET_FS_unindex_do_remove_kblocks_ (uc);
     return;
   }
-  GNUNET_CRYPTO_hash (data,
-                      size,
-                      &dh);
-  if (GNUNET_YES ==
-      GNUNET_CONTAINER_multihashmap_contains (uc->seen_dh,
-                                              &dh))
-  {
-    GNUNET_CONTAINER_multihashmap_clear (uc->seen_dh);
-    uc->ksk_offset++;
-    GNUNET_FS_unindex_do_remove_kblocks_ (uc);
-    return;
-  }
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONTAINER_multihashmap_put (uc->seen_dh,
-                                                    &dh,
-                                                    uc,
-                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   GNUNET_assert (GNUNET_BLOCK_TYPE_FS_UBLOCK == type);
   if (size < sizeof (struct UBlock))
   {
@@ -565,23 +544,24 @@ process_kblock_for_unindex (void *cls,
   GNUNET_FS_uri_destroy (chk_uri);
   /* matches! */
   uc->dqe = GNUNET_DATASTORE_remove (uc->dsh,
-                                     key,
+                                     key,
                                      size,
                                      data,
-                                     0 /* priority */,
+                                     0 /* priority */,
                                      1 /* queue size */,
-                                     &continue_after_remove,
-                                     uc);
+                                     &continue_after_remove,
+                                     uc);
   return;
  get_next:
   uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
-                                      uc->roff++,
-                                      &uc->uquery,
-                                      GNUNET_BLOCK_TYPE_FS_UBLOCK,
-                                      0 /* priority */,
+                                      uid + 1 /* next_uid */,
+                                      false /* random */,
+                                      &uc->uquery,
+                                      GNUNET_BLOCK_TYPE_FS_UBLOCK,
+                                      0 /* priority */,
                                       1 /* queue size */,
-                                      &process_kblock_for_unindex,
-                                      uc);
+                                      &process_kblock_for_unindex,
+                                      uc);
 }
 
 
@@ -626,13 +606,14 @@ GNUNET_FS_unindex_do_remove_kblocks_ (struct GNUNET_FS_UnindexContext *uc)
                       sizeof (dpub),
                       &uc->uquery);
   uc->dqe = GNUNET_DATASTORE_get_key (uc->dsh,
-                                      uc->roff++,
-                                      &uc->uquery,
-                                      GNUNET_BLOCK_TYPE_FS_UBLOCK,
-                                      0 /* priority */,
+                                      0 /* next_uid */,
+                                      false /* random */,
+                                      &uc->uquery,
+                                      GNUNET_BLOCK_TYPE_FS_UBLOCK,
+                                      0 /* priority */,
                                       1 /* queue size */,
-                                      &process_kblock_for_unindex,
-                                      uc);
+                                      &process_kblock_for_unindex,
+                                      uc);
 }
 
 
@@ -825,8 +806,6 @@ GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h,
   uc->start_time = GNUNET_TIME_absolute_get ();
   uc->file_size = size;
   uc->client_info = cctx;
-  uc->seen_dh = GNUNET_CONTAINER_multihashmap_create (4,
-                                                       GNUNET_NO);
   GNUNET_FS_unindex_sync_ (uc);
   pi.status = GNUNET_FS_STATUS_UNINDEX_START;
   pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL;
diff --git a/src/fs/fs_uri.c b/src/fs/fs_uri.c
index fa27e6e9b..11968b750 100644
--- a/src/fs/fs_uri.c
+++ b/src/fs/fs_uri.c
@@ -309,7 +309,8 @@ uri_ksk_parse (const char *s,
   }
   iret = max;
   dup = GNUNET_strdup (s);
-  keywords = GNUNET_malloc (max * sizeof (char *));
+  keywords = GNUNET_new_array (max,
+                               char *);
   for (i = slen - 1; i >= (int) pos; i--)
   {
     if ((s[i] == '%') && (&s[i] == strstr (&s[i], "%22")))
@@ -937,7 +938,8 @@ GNUNET_FS_uri_ksk_merge (const struct GNUNET_FS_Uri *u1,
     return NULL;
   }
   kc = u1->data.ksk.keywordCount;
-  kl = GNUNET_malloc ((kc + u2->data.ksk.keywordCount) * sizeof (char *));
+  kl = GNUNET_new_array (kc + u2->data.ksk.keywordCount,
+                         char *);
   for (i = 0; i < u1->data.ksk.keywordCount; i++)
     kl[i] = GNUNET_strdup (u1->data.ksk.keywords[i]);
   for (i = 0; i < u2->data.ksk.keywordCount; i++)
@@ -991,8 +993,9 @@ GNUNET_FS_uri_dup (const struct GNUNET_FS_Uri *uri)
     }
     if (ret->data.ksk.keywordCount > 0)
     {
-      ret->data.ksk.keywords =
-          GNUNET_malloc (ret->data.ksk.keywordCount * sizeof (char *));
+      ret->data.ksk.keywords
+        = GNUNET_new_array (ret->data.ksk.keywordCount,
+                            char *);
       for (i = 0; i < ret->data.ksk.keywordCount; i++)
         ret->data.ksk.keywords[i] = GNUNET_strdup (uri->data.ksk.keywords[i]);
     }
@@ -1078,7 +1081,8 @@ GNUNET_FS_uri_ksk_create (const char *keywords,
     *emsg = GNUNET_strdup (_("Number of double-quotes not balanced!\n"));
     return NULL;
   }
-  keywordarr = GNUNET_malloc (num_Words * sizeof (char *));
+  keywordarr = GNUNET_new_array (num_Words,
+                                 char *);
   num_Words = 0;
   inWord = 0;
   pos = searchString;
@@ -1151,7 +1155,8 @@ GNUNET_FS_uri_ksk_create_from_args (unsigned int argc,
   uri = GNUNET_new (struct GNUNET_FS_Uri);
   uri->type = GNUNET_FS_URI_KSK;
   uri->data.ksk.keywordCount = argc;
-  uri->data.ksk.keywords = GNUNET_malloc (argc * sizeof (char *));
+  uri->data.ksk.keywords = GNUNET_new_array (argc,
+                                             char *);
   for (i = 0; i < argc; i++)
   {
     keyword = argv[i];
@@ -1766,8 +1771,9 @@ GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_CONTAINER_MetaData
     }
     /* x3 because there might be a normalized variant of every keyword,
        plus theoretically one more for mime... */
-    ret->data.ksk.keywords = GNUNET_malloc
-      (sizeof (char *) * (ent + tok_keywords + paren_keywords) * 3);
+    ret->data.ksk.keywords
+      = GNUNET_new_array ((ent + tok_keywords + paren_keywords) * 3,
+                          char *);
     GNUNET_CONTAINER_meta_data_iterate (md, &gather_uri_data, ret);
   }
   if (tok_keywords > 0)
diff --git a/src/fs/gnunet-auto-share.c b/src/fs/gnunet-auto-share.c
index cc0111111..2f980520a 100644
--- a/src/fs/gnunet-auto-share.c
+++ b/src/fs/gnunet-auto-share.c
@@ -72,7 +72,7 @@ static int ret;
 /**
  * Are we running 'verbosely'?
  */
-static int verbose;
+static unsigned int verbose;
 
 /**
  * Configuration to use.
@@ -759,26 +759,38 @@ free_item (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "anonymity", "LEVEL",
-     gettext_noop ("set the desired LEVEL of sender-anonymity"),
-     1, &GNUNET_GETOPT_set_uint, &anonymity_level},
-    {'d', "disable-creation-time", NULL,
-     gettext_noop
-     ("disable adding the creation time to the metadata of the uploaded file"),
-     0, &GNUNET_GETOPT_set_one, &do_disable_creation_time},
-    {'D', "disable-extractor", NULL,
-     gettext_noop ("do not use libextractor to add keywords or metadata"),
-     0, &GNUNET_GETOPT_set_one, &disable_extractor},
-    {'p', "priority", "PRIORITY",
-     gettext_noop ("specify the priority of the content"),
-     1, &GNUNET_GETOPT_set_uint, &content_priority},
-    {'r', "replication", "LEVEL",
-     gettext_noop ("set the desired replication LEVEL"),
-     1, &GNUNET_GETOPT_set_uint, &replication_level},
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_set_one, &verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('a',
+                                   "anonymity",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired LEVEL of sender-anonymity"),
+                                   &anonymity_level),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('d',
+                                  "disable-creation-time",
+                                  gettext_noop ("disable adding the creation time to the metadata of the uploaded file"),
+                                  &do_disable_creation_time),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('D',
+                                  "disable-extractor",
+                                  gettext_noop ("do not use libextractor to add keywords or metadata"),
+                                  &disable_extractor),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('p',
+                                   "priority",
+                                   "PRIORITY",
+                                   gettext_noop ("specify the priority of the content"),
+                                   &content_priority),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('r',
+                                   "replication",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired replication LEVEL"),
+                                   &replication_level),
+
+    GNUNET_GETOPT_OPTION_VERBOSE (&verbose),
+
     GNUNET_GETOPT_OPTION_END
   };
   struct WorkItem *wi;
diff --git a/src/fs/gnunet-download.c b/src/fs/gnunet-download.c
index 6d9adb8ab..4d6f30587 100644
--- a/src/fs/gnunet-download.c
+++ b/src/fs/gnunet-download.c
@@ -30,7 +30,7 @@
 
 static int ret;
 
-static int verbose;
+static unsigned int verbose;
 
 static int delete_incomplete;
 
@@ -299,33 +299,51 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "anonymity", "LEVEL",
-     gettext_noop ("set the desired LEVEL of receiver-anonymity"),
-     1, &GNUNET_GETOPT_set_uint, &anonymity},
-    {'D', "delete-incomplete", NULL,
-     gettext_noop ("delete incomplete downloads (when aborted with CTRL-C)"),
-     0, &GNUNET_GETOPT_set_one, &delete_incomplete},
-    {'n', "no-network", NULL,
-     gettext_noop ("only search the local peer (no P2P network search)"),
-     0, &GNUNET_GETOPT_set_one, &local_only},
-    {'o', "output", "FILENAME",
-     gettext_noop ("write the file to FILENAME"),
-     1, &GNUNET_GETOPT_set_string, &filename},
-    {'p', "parallelism", "DOWNLOADS",
-     gettext_noop
-     ("set the maximum number of parallel downloads that is allowed"),
-     1, &GNUNET_GETOPT_set_uint, &parallelism},
-    {'r', "request-parallelism", "REQUESTS",
-     gettext_noop
-     ("set the maximum number of parallel requests for blocks that is allowed"),
-     1, &GNUNET_GETOPT_set_uint, &request_parallelism},
-    {'R', "recursive", NULL,
-     gettext_noop ("download a GNUnet directory recursively"),
-     0, &GNUNET_GETOPT_set_one, &do_recursive},
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_increment_value, &verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_UINT ('a',
+                                   "anonymity",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired LEVEL of receiver-anonymity"),
+                                   &anonymity),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('D',
+                                  "delete-incomplete",
+                                  gettext_noop ("delete incomplete downloads (when aborted with CTRL-C)"),
+                                  &delete_incomplete),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "no-network",
+                                  gettext_noop ("only search the local peer (no P2P network search)"),
+                                  &local_only), 
+
+    GNUNET_GETOPT_OPTION_STRING ('o',
+                                 "output",
+                                 "FILENAME",
+                                 gettext_noop ("write the file to FILENAME"),
+                                 &filename),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('p',
+                                   "parallelism",
+                                   "DOWNLOADS",
+                                   gettext_noop ("set the maximum number of parallel downloads that is allowed"),
+                                   &parallelism),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('r',
+                                   "request-parallelism",
+                                   "REQUESTS",
+                                   gettext_noop ("set the maximum number of parallel requests for blocks that is allowed"),
+                                   &request_parallelism), 
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('R',
+                                  "recursive",
+                                  gettext_noop ("download a GNUnet directory recursively"),
+                                  &do_recursive),
+
+    GNUNET_GETOPT_OPTION_INCREMENT_VALUE ('V',
+                                          "verbose",
+                                          gettext_noop ("be verbose (print progress information)"),
+                                          &verbose), 
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/fs/gnunet-fs-profiler.c b/src/fs/gnunet-fs-profiler.c
index cfbe57bbd..fb99d8f90 100644
--- a/src/fs/gnunet-fs-profiler.c
+++ b/src/fs/gnunet-fs-profiler.c
@@ -203,16 +203,26 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'n', "num-peers", "COUNT",
-     gettext_noop ("run the experiment with COUNT peers"),
-     1, &GNUNET_GETOPT_set_uint, &num_peers},
-    {'H', "hosts", "HOSTFILE",
-     gettext_noop ("specifies name of a file with the HOSTS the testbed should use"),
-     1, &GNUNET_GETOPT_set_string, &host_filename},
-    {'t', "timeout", "DELAY",
-     gettext_noop ("automatically terminate experiment after DELAY"),
-     1, &GNUNET_GETOPT_set_relative_time, &timeout},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('n',
+                                   "num-peers",
+                                   "COUNT",
+                                   gettext_noop ("run the experiment with COUNT peers"),
+                                   &num_peers),
+
+    GNUNET_GETOPT_OPTION_STRING ('H',
+                                 "hosts",
+                                 "HOSTFILE",
+                                 gettext_noop ("specifies name of a file with the HOSTS the testbed should use"),
+                                 &host_filename),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('t',
+                                            "timeout",
+                                            "DELAY",
+                                            gettext_noop ("automatically terminate experiment after DELAY"),
+                                            &timeout),
+
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/fs/gnunet-fs.c b/src/fs/gnunet-fs.c
index 7c20e025f..2b24b7124 100644
--- a/src/fs/gnunet-fs.c
+++ b/src/fs/gnunet-fs.c
@@ -43,7 +43,7 @@ static int list_indexed_files;
 /**
  * Option -v given?
  */
-static int verbose;
+static unsigned int verbose;
 
 
 /**
@@ -112,10 +112,13 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'i', "list-indexed", NULL,
-     gettext_noop ("print a list of all indexed files"), 0,
-     &GNUNET_GETOPT_set_one, &list_indexed_files},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('i',
+                                  "list-indexed",
+                                  gettext_noop ("print a list of all indexed files"),
+                                  &list_indexed_files),
+
     GNUNET_GETOPT_OPTION_VERBOSE (&verbose),
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/fs/gnunet-helper-fs-publish.c b/src/fs/gnunet-helper-fs-publish.c
index 55836e772..72a656de1 100644
--- a/src/fs/gnunet-helper-fs-publish.c
+++ b/src/fs/gnunet-helper-fs-publish.c
@@ -480,7 +480,7 @@ make_dev_zero (int fd,
   GNUNET_assert (-1 != z);
   if (z == fd)
     return;
-  dup2 (z, fd);
+  GNUNET_break (fd == dup2 (z, fd));
   GNUNET_assert (0 == close (z));
 }
 
diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c
index a563d7b7a..2229e45e7 100644
--- a/src/fs/gnunet-publish.c
+++ b/src/fs/gnunet-publish.c
@@ -37,7 +37,7 @@ static int ret;
 /**
  * Command line option 'verbose' set
  */
-static int verbose;
+static unsigned int verbose;
 
 /**
  * Handle to our configuration.
@@ -893,63 +893,98 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "anonymity", "LEVEL",
-     gettext_noop ("set the desired LEVEL of sender-anonymity"),
-     1, &GNUNET_GETOPT_set_uint, &bo.anonymity_level},
-    {'d', "disable-creation-time", NULL,
-     gettext_noop
-     ("disable adding the creation time to the metadata of the uploaded file"),
-     0, &GNUNET_GETOPT_set_one, &do_disable_creation_time},
-    {'D', "disable-extractor", NULL,
-     gettext_noop ("do not use libextractor to add keywords or metadata"),
-     0, &GNUNET_GETOPT_set_one, &disable_extractor},
-    {'e', "extract", NULL,
-     gettext_noop
-     ("print list of extracted keywords that would be used, but do not perform upload"),
-     0, &GNUNET_GETOPT_set_one, &extract_only},
-    {'k', "key", "KEYWORD",
-     gettext_noop
-     ("add an additional keyword for the top-level file or directory"
-      " (this option can be specified multiple times)"),
-     1, &GNUNET_FS_getopt_set_keywords, &topKeywords},
-    {'m', "meta", "TYPE:VALUE",
-     gettext_noop ("set the meta-data for the given TYPE to the given VALUE"),
-     1, &GNUNET_FS_getopt_set_metadata, &meta},
-    {'n', "noindex", NULL,
-     gettext_noop ("do not index, perform full insertion (stores entire "
-                   "file in encrypted form in GNUnet database)"),
-     0, &GNUNET_GETOPT_set_one, &do_insert},
-    {'N', "next", "ID",
-     gettext_noop
-     ("specify ID of an updated version to be published in the future"
-      " (for namespace insertions only)"),
-     1, &GNUNET_GETOPT_set_string, &next_id},
-    {'p', "priority", "PRIORITY",
-     gettext_noop ("specify the priority of the content"),
-     1, &GNUNET_GETOPT_set_uint, &bo.content_priority},
-    {'P', "pseudonym", "NAME",
-     gettext_noop
-     ("publish the files under the pseudonym NAME (place file into namespace)"),
-     1, &GNUNET_GETOPT_set_string, &pseudonym},
-    {'r', "replication", "LEVEL",
-     gettext_noop ("set the desired replication LEVEL"),
-     1, &GNUNET_GETOPT_set_uint, &bo.replication_level},
-    {'s', "simulate-only", NULL,
-     gettext_noop ("only simulate the process but do not do any "
-                   "actual publishing (useful to compute URIs)"),
-     0, &GNUNET_GETOPT_set_one, &do_simulate},
-    {'t', "this", "ID",
-     gettext_noop ("set the ID of this version of the publication"
-                   " (for namespace insertions only)"),
-     1, &GNUNET_GETOPT_set_string, &this_id},
-    {'u', "uri", "URI",
-     gettext_noop ("URI to be published (can be used instead of passing a "
-                   "file to add keywords to the file with the respective URI)"),
-     1, &GNUNET_GETOPT_set_string, &uri_string},
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_set_one, &verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_UINT ('a',
+                                   "anonymity",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired LEVEL of sender-anonymity"),
+                                   &bo.anonymity_level),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('d',
+                                  "disable-creation-time",
+                                  gettext_noop ("disable adding the creation time to the "
+                                                "metadata of the uploaded file"),
+                                  &do_disable_creation_time),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('D',
+                                  "disable-extractor",
+                                  gettext_noop ("do not use libextractor to add keywords or metadata"),
+                                  &disable_extractor),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('e',
+                                  "extract",
+                                  gettext_noop ("print list of extracted keywords that would "
+                                                "be used, but do not perform upload"),
+                                  &extract_only),
+
+    GNUNET_FS_GETOPT_KEYWORDS ('k',
+                               "key",
+                               "KEYWORD",
+                               gettext_noop ("add an additional keyword for the top-level "
+                                             "file or directory (this option can be specified multiple times)"),
+                               &topKeywords),
+
+    GNUNET_FS_GETOPT_METADATA ('m',
+                               "meta",
+                               "TYPE:VALUE",
+                               gettext_noop ("set the meta-data for the given TYPE to the given VALUE"),
+                               &meta),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "noindex",
+                                  gettext_noop ("do not index, perform full insertion (stores "
+                                                "entire file in encrypted form in GNUnet database)"),
+                                  &do_insert),
+
+    GNUNET_GETOPT_OPTION_STRING ('N',
+                                 "next",
+                                 "ID",
+                                 gettext_noop ("specify ID of an updated version to be "
+                                               "published in the future (for namespace insertions only)"),
+                                 &next_id),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('p',
+                                   "priority",
+                                   "PRIORITY",
+                                   gettext_noop ("specify the priority of the content"),
+                                   &bo.content_priority),
+
+    GNUNET_GETOPT_OPTION_STRING ('P',
+                                 "pseudonym",
+                                 "NAME",
+                                 gettext_noop ("publish the files under the pseudonym "
+                                               "NAME (place file into namespace)"),
+                                 &pseudonym),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('r',
+                                   "replication",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired replication LEVEL"),
+                                   &bo.replication_level),
+
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('s',
+                                  "simulate-only",
+                                  gettext_noop ("only simulate the process but do not do "
+                                                "any actual publishing (useful to compute URIs)"),
+                                  &do_simulate),
+
+    GNUNET_GETOPT_OPTION_STRING ('t',
+                                 "this",
+                                 "ID",
+                                 gettext_noop ("set the ID of this version of the publication "
+                                               "(for namespace insertions only)"),
+                                 &this_id),
+
+    GNUNET_GETOPT_OPTION_STRING ('u',
+                                 "uri",
+                                 "URI",
+                                 gettext_noop ("URI to be published (can be used instead of passing a "
+                                               "file to add keywords to the file with the respective URI)"),
+                                 &uri_string), 
+
+    GNUNET_GETOPT_OPTION_VERBOSE (&verbose),
+
     GNUNET_GETOPT_OPTION_END
   };
   bo.expiration_time =
diff --git a/src/fs/gnunet-search.c b/src/fs/gnunet-search.c
index dfe6d0e75..22e790cf3 100644
--- a/src/fs/gnunet-search.c
+++ b/src/fs/gnunet-search.c
@@ -51,7 +51,7 @@ static unsigned int results_limit;
 
 static unsigned int results;
 
-static int verbose;
+static unsigned int verbose;
 
 static int local_only;
 
@@ -305,26 +305,42 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "anonymity", "LEVEL",
-     gettext_noop ("set the desired LEVEL of receiver-anonymity"),
-     1, &GNUNET_GETOPT_set_uint, &anonymity},
-    {'n', "no-network", NULL,
-     gettext_noop ("only search the local peer (no P2P network search)"),
-     0, &GNUNET_GETOPT_set_one, &local_only},
-    {'o', "output", "PREFIX",
-     gettext_noop ("write search results to file starting with PREFIX"),
-     1, &GNUNET_GETOPT_set_string, &output_filename},
-    {'t', "timeout", "DELAY",
-     gettext_noop ("automatically terminate search after DELAY"),
-     1, &GNUNET_GETOPT_set_relative_time, &timeout},
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_set_one, &verbose},
-    {'N', "results", "VALUE",
-     gettext_noop
-     ("automatically terminate search after VALUE results are found"),
-     1, &GNUNET_GETOPT_set_uint, &results_limit},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('a',
+                                   "anonymity",
+                                   "LEVEL",
+                                   gettext_noop ("set the desired LEVEL of receiver-anonymity"),
+                                   &anonymity),
+
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "no-network",
+                                  gettext_noop ("only search the local peer (no P2P network search)"),
+                                  &local_only),
+                                  
+    GNUNET_GETOPT_OPTION_STRING ('o',
+                                 "output",
+                                 "PREFIX",
+                                 gettext_noop ("write search results to file starting with PREFIX"),
+                                 &output_filename),                              
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('t', 
+                                            "timeout",
+                                            "DELAY",
+                                            gettext_noop ("automatically terminate search after DELAY"),
+                                            &timeout),
+
+
+    GNUNET_GETOPT_OPTION_VERBOSE (&verbose),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('N',
+                                   "results",
+                                   "VALUE",
+                                   gettext_noop ("automatically terminate search "
+                                                 "after VALUE results are found"),
+                                   &results_limit),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c
index 256d0c2b8..09b1e05d8 100644
--- a/src/fs/gnunet-service-fs.c
+++ b/src/fs/gnunet-service-fs.c
@@ -38,7 +38,6 @@
 #include "gnunet_util_lib.h"
 #include "gnunet-service-fs_cp.h"
 #include "gnunet-service-fs_indexing.h"
-#include "gnunet-service-fs_lc.h"
 #include "gnunet-service-fs_pe.h"
 #include "gnunet-service-fs_pr.h"
 #include "gnunet-service-fs_push.h"
diff --git a/src/fs/gnunet-service-fs_cadet_client.c b/src/fs/gnunet-service-fs_cadet_client.c
index 55e0cbc24..c729ebe41 100644
--- a/src/fs/gnunet-service-fs_cadet_client.c
+++ b/src/fs/gnunet-service-fs_cadet_client.c
@@ -487,7 +487,7 @@ reset_cadet (struct CadetHandle *mh)
     GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
                         strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
                         &port);
-    mh->channel = GNUNET_CADET_channel_creatE (cadet_handle,
+    mh->channel = GNUNET_CADET_channel_create (cadet_handle,
                                                mh,
                                                &mh->target,
                                                &port,
@@ -627,7 +627,7 @@ get_cadet (const struct GNUNET_PeerIdentity *target)
     GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
                         strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
                         &port);
-    mh->channel = GNUNET_CADET_channel_creatE (cadet_handle,
+    mh->channel = GNUNET_CADET_channel_create (cadet_handle,
                                                mh,
                                                &mh->target,
                                                &port,
diff --git a/src/fs/gnunet-service-fs_cadet_server.c b/src/fs/gnunet-service-fs_cadet_server.c
index adbce1154..f8619b812 100644
--- a/src/fs/gnunet-service-fs_cadet_server.c
+++ b/src/fs/gnunet-service-fs_cadet_server.c
@@ -289,7 +289,7 @@ handle_datastore_reply (void *cls,
     }
     return;
   }
-  if (msize > GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msize > GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     continue_writing (sc);
@@ -345,12 +345,13 @@ handle_request (void *cls,
                             GNUNET_NO);
   refresh_timeout_task (sc);
   sc->qe = GNUNET_DATASTORE_get_key (GSF_dsh,
-                                     0,
-                                     &sqm->query,
-                                     ntohl (sqm->type),
-                                     0 /* priority */,
-                                     GSF_datastore_queue_size,
-                                     &handle_datastore_reply,
+                                     0 /* next_uid */,
+                                     false /* random */,
+                                     &sqm->query,
+                                     ntohl (sqm->type),
+                                     0 /* priority */,
+                                     GSF_datastore_queue_size,
+                                     &handle_datastore_reply,
                                      sc);
   if (NULL == sc->qe)
   {
@@ -499,12 +500,12 @@ GSF_cadet_start_server ()
               "Initializing cadet FS server with a limit of %llu connections\n",
               sc_count_max);
   cadet_map = GNUNET_CONTAINER_multipeermap_create (16, GNUNET_YES);
-  cadet_handle = GNUNET_CADET_connecT (GSF_cfg);
+  cadet_handle = GNUNET_CADET_connect (GSF_cfg);
   GNUNET_assert (NULL != cadet_handle);
   GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER,
                       strlen (GNUNET_APPLICATION_PORT_FS_BLOCK_TRANSFER),
                       &port);
-  cadet_port = GNUNET_CADET_open_porT (cadet_handle,
+  cadet_port = GNUNET_CADET_open_port (cadet_handle,
                                        &port,
                                        &connect_cb,
                                        NULL,
diff --git a/src/fs/gnunet-service-fs_cp.c b/src/fs/gnunet-service-fs_cp.c
index 3f7783ded..817aed257 100644
--- a/src/fs/gnunet-service-fs_cp.c
+++ b/src/fs/gnunet-service-fs_cp.c
@@ -859,7 +859,7 @@ handle_p2p_reply (void *cls,
   size_t msize;
 
   GNUNET_assert (data_len + sizeof (struct PutMessage) <
-                 GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                 GNUNET_MAX_MESSAGE_SIZE);
   GNUNET_assert (peerreq->pr == pr);
   prd = GSF_pending_request_get_data_ (pr);
   if (NULL == data)
@@ -883,7 +883,7 @@ handle_p2p_reply (void *cls,
                             gettext_noop ("# replies received for other peers"),
                             1, GNUNET_NO);
   msize = sizeof (struct PutMessage) + data_len;
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msize >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
diff --git a/src/fs/gnunet-service-fs_indexing.c b/src/fs/gnunet-service-fs_indexing.c
index 3ce68f487..ce6bcec41 100644
--- a/src/fs/gnunet-service-fs_indexing.c
+++ b/src/fs/gnunet-service-fs_indexing.c
@@ -438,7 +438,7 @@ GNUNET_FS_indexing_send_list (struct GNUNET_MQ_Handle *mq)
     fn = pos->filename;
     slen = strlen (fn) + 1;
     if (slen + sizeof (struct IndexInfoMessage) >=
-        GNUNET_SERVER_MAX_MESSAGE_SIZE)
+        GNUNET_MAX_MESSAGE_SIZE)
     {
       GNUNET_break (0);
       break;
diff --git a/src/fs/gnunet-service-fs_lc.c b/src/fs/gnunet-service-fs_lc.c
deleted file mode 100644
index 9ffd6cadd..000000000
--- a/src/fs/gnunet-service-fs_lc.c
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file fs/gnunet-service-fs_lc.c
- * @brief API to handle 'local clients'
- * @author Christian Grothoff
- */
-#include "platform.h"
-#include "gnunet-service-fs.h"
-#include "gnunet-service-fs_lc.h"
-#include "gnunet-service-fs_cp.h"
-#include "gnunet-service-fs_pr.h"
-
-
-
-/* end of gnunet-service-fs_lc.c */
diff --git a/src/fs/gnunet-service-fs_lc.h b/src/fs/gnunet-service-fs_lc.h
deleted file mode 100644
index 6671ed33c..000000000
--- a/src/fs/gnunet-service-fs_lc.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file fs/gnunet-service-fs_lc.h
- * @brief API to handle 'local clients'
- * @author Christian Grothoff
- */
-#ifndef GNUNET_SERVICE_FS_LC_H
-#define GNUNET_SERVICE_FS_LC_H
-
-#include "gnunet-service-fs.h"
-
-
-#endif
-/* end of gnunet-service-fs_lc.h */
diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c
index b0fda24b5..b736b49c2 100644
--- a/src/fs/gnunet-service-fs_pr.c
+++ b/src/fs/gnunet-service-fs_pr.c
@@ -160,20 +160,27 @@ struct GSF_PendingRequest
   struct GNUNET_SCHEDULER_Task * warn_task;
 
   /**
-   * Current offset for querying our local datastore for results.
-   * Starts at a random value, incremented until we get the same
-   * UID again (detected using 'first_uid'), which is then used
-   * to termiante the iteration.
+   * Do we have a first UID yet?
+   */
+  bool have_first_uid;
+
+  /**
+   * Have we seen a NULL result yet?
    */
-  uint64_t local_result_offset;
+  bool seen_null;
 
   /**
    * Unique ID of the first result from the local datastore;
-   * used to detect wrap-around of the offset.
+   * used to terminate the loop.
    */
   uint64_t first_uid;
 
   /**
+   * Result count.
+   */
+  size_t result_count;
+
+  /**
    * How often have we retried this request via 'cadet'?
    * (used to bound overall retries).
    */
@@ -189,11 +196,6 @@ struct GSF_PendingRequest
    */
   unsigned int replies_seen_size;
 
-  /**
-   * Do we have a first UID yet?
-   */
-  unsigned int have_first_uid;
-
 };
 
 
@@ -332,8 +334,6 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options,
   if (NULL != target)
     extra += sizeof (struct GNUNET_PeerIdentity);
   pr = GNUNET_malloc (sizeof (struct GSF_PendingRequest) + extra);
-  pr->local_result_offset =
-      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX);
   pr->public_data.query = *query;
   eptr = (struct GNUNET_HashCode *) &pr[1];
   if (NULL != target)
@@ -1340,6 +1340,123 @@ odc_warn_delay_task (void *cls)
 }
 
 
+/* Call our continuation (if we have any) */
+static void
+call_continuation (struct GSF_PendingRequest *pr)
+{
+  GSF_LocalLookupContinuation cont = pr->llc_cont;
+
+  GNUNET_assert (NULL == pr->qe);
+  if (NULL != pr->warn_task)
+  {
+    GNUNET_SCHEDULER_cancel (pr->warn_task);
+    pr->warn_task = NULL;
+  }
+  if (NULL == cont)
+    return;                     /* no continuation */
+  pr->llc_cont = NULL;
+  if (0 != (GSF_PRO_LOCAL_ONLY & pr->public_data.options))
+  {
+    if (GNUNET_BLOCK_EVALUATION_OK_LAST != pr->local_result)
+    {
+      /* Signal that we are done and that there won't be any
+         additional results to allow client to clean up state. */
+      pr->rh (pr->rh_cls,
+               GNUNET_BLOCK_EVALUATION_OK_LAST,
+               pr,
+               UINT32_MAX,
+               GNUNET_TIME_UNIT_ZERO_ABS,
+               GNUNET_TIME_UNIT_FOREVER_ABS,
+               GNUNET_BLOCK_TYPE_ANY,
+               NULL,
+               0);
+    }
+    /* Finally, call our continuation to signal that we are
+       done with local processing of this request; i.e. to
+       start reading again from the client. */
+    cont (pr->llc_cont_cls, NULL, GNUNET_BLOCK_EVALUATION_OK_LAST);
+    return;
+  }
+
+  cont (pr->llc_cont_cls, pr, pr->local_result);
+}
+
+
+/* Update stats and call continuation */
+static void
+no_more_local_results (struct GSF_PendingRequest *pr)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
+              "No further local responses available.\n");
+#if INSANE_STATISTICS
+  if ( (GNUNET_BLOCK_TYPE_FS_DBLOCK == pr->public_data.type) ||
+       (GNUNET_BLOCK_TYPE_FS_IBLOCK == pr->public_data.type) )
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop ("# requested DBLOCK or IBLOCK not found"),
+                              1,
+                              GNUNET_NO);
+#endif
+  call_continuation (pr);
+}
+
+
+/* forward declaration */
+static void
+process_local_reply (void *cls,
+                     const struct GNUNET_HashCode *key,
+                     size_t size,
+                     const void *data,
+                     enum GNUNET_BLOCK_Type type,
+                     uint32_t priority,
+                     uint32_t anonymity,
+                     struct GNUNET_TIME_Absolute expiration,
+                     uint64_t uid);
+
+
+/* Start a local query */
+static void
+start_local_query (struct GSF_PendingRequest *pr,
+                   uint64_t next_uid,
+                   bool random)
+{
+  pr->qe_start = GNUNET_TIME_absolute_get ();
+  pr->warn_task =
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+                                    &warn_delay_task,
+                                    pr);
+  pr->qe =
+      GNUNET_DATASTORE_get_key (GSF_dsh,
+                                next_uid,
+                                random,
+                                &pr->public_data.query,
+                                pr->public_data.type ==
+                                GNUNET_BLOCK_TYPE_FS_DBLOCK ?
+                                GNUNET_BLOCK_TYPE_ANY : pr->public_data.type,
+                                (0 !=
+                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
+                                  public_data.options)) ? UINT_MAX : 1
+                                /* queue priority */ ,
+                                (0 !=
+                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
+                                  public_data.options)) ? UINT_MAX :
+                                GSF_datastore_queue_size
+                                /* max queue size */ ,
+                                &process_local_reply, pr);
+  if (NULL != pr->qe)
+    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "ERROR Requesting `%s' of type %d with next_uid %llu from datastore.\n",
+              GNUNET_h2s (&pr->public_data.query),
+              pr->public_data.type,
+              (unsigned long long) next_uid);
+  GNUNET_STATISTICS_update (GSF_stats,
+                            gettext_noop ("# Datastore lookups concluded (error queueing)"),
+                            1,
+                            GNUNET_NO);
+  call_continuation (pr);
+}
+
+
 /**
  * We're processing (local) results for a search request
  * from another peer.  Pass applicable results to the
@@ -1369,69 +1486,71 @@ process_local_reply (void *cls,
                      uint64_t uid)
 {
   struct GSF_PendingRequest *pr = cls;
-  GSF_LocalLookupContinuation cont;
   struct ProcessReplyClosure prq;
   struct GNUNET_HashCode query;
   unsigned int old_rf;
 
   GNUNET_SCHEDULER_cancel (pr->warn_task);
   pr->warn_task = NULL;
-  if (NULL != pr->qe)
+  if (NULL == pr->qe)
+    goto called_from_on_demand;
+  pr->qe = NULL;
+  if ( (NULL == key) &&
+       pr->seen_null &&
+       !pr->have_first_uid) /* We have hit the end for the 2nd time with no results */
   {
-    pr->qe = NULL;
-    if (NULL == key)
-    {
+    /* No results */
 #if INSANE_STATISTICS
-      GNUNET_STATISTICS_update (GSF_stats,
-                                gettext_noop
-                                ("# Datastore lookups concluded (no results)"),
-                                1, GNUNET_NO);
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                              ("# Datastore lookups concluded (no results)"),
+                              1, GNUNET_NO);
 #endif
-    }
-    if (GNUNET_NO == pr->have_first_uid)
-    {
-      pr->first_uid = uid;
-      pr->have_first_uid = 1;
-    }
-    else
-    {
-      if ((uid == pr->first_uid) && (key != NULL))
-      {
-        GNUNET_STATISTICS_update (GSF_stats,
-                                  gettext_noop
-                                  ("# Datastore lookups concluded (seen all)"),
-                                  1, GNUNET_NO);
-        key = NULL;             /* all replies seen! */
-      }
-      pr->have_first_uid++;
-      if ((pr->have_first_uid > MAX_RESULTS) && (key != NULL))
-      {
-        GNUNET_STATISTICS_update (GSF_stats,
-                                  gettext_noop
-                                  ("# Datastore lookups aborted (more than MAX_RESULTS)"),
-                                  1, GNUNET_NO);
-        key = NULL;             /* all replies seen! */
-      }
-    }
+    no_more_local_results (pr);
+    return;
+  }
+  if ( ( (NULL == key) &&
+         pr->seen_null ) || /* We have hit the end for the 2nd time OR */
+       ( pr->seen_null &&
+         pr->have_first_uid &&
+         (uid >= pr->first_uid) ) ) /* We have hit the end and past first UID */
+  {
+    /* Seen all results */
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                              ("# Datastore lookups concluded (seen all)"),
+                              1, GNUNET_NO);
+    no_more_local_results (pr);
+    return;
   }
   if (NULL == key)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
-                "No further local responses available.\n");
-#if INSANE_STATISTICS
-    if ((pr->public_data.type == GNUNET_BLOCK_TYPE_FS_DBLOCK) ||
-        (pr->public_data.type == GNUNET_BLOCK_TYPE_FS_IBLOCK))
-      GNUNET_STATISTICS_update (GSF_stats,
-                                gettext_noop
-                                ("# requested DBLOCK or IBLOCK not found"), 1,
-                                GNUNET_NO);
-#endif
-    goto check_error_and_continue;
+    GNUNET_assert (!pr->seen_null);
+    pr->seen_null = true;
+    start_local_query (pr,
+                       0 /* next_uid */,
+                       false /* random */);
+    return;
+  }
+  if (!pr->have_first_uid)
+  {
+    pr->first_uid = uid;
+    pr->have_first_uid = true;
+  }
+  pr->result_count++;
+  if (pr->result_count > MAX_RESULTS)
+  {
+    GNUNET_STATISTICS_update (GSF_stats,
+                              gettext_noop
+                              ("# Datastore lookups aborted (more than MAX_RESULTS)"),
+                              1, GNUNET_NO);
+    no_more_local_results (pr);
+    return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received reply for `%s' of type %d with UID %llu from datastore.\n",
               GNUNET_h2s (key), type, (unsigned long long) uid);
-  if (type == GNUNET_BLOCK_TYPE_FS_ONDEMAND)
+  if (GNUNET_BLOCK_TYPE_FS_ONDEMAND == type)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Found ONDEMAND block, performing on-demand encoding\n");
@@ -1458,33 +1577,12 @@ process_local_reply (void *cls,
                               gettext_noop ("# on-demand lookups failed"), 1,
                               GNUNET_NO);
     GNUNET_SCHEDULER_cancel (pr->warn_task);
-    pr->warn_task =
-        GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                      &warn_delay_task, pr);
-    pr->qe =
-        GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset - 1,
-                                  &pr->public_data.query,
-                                  pr->public_data.type ==
-                                  GNUNET_BLOCK_TYPE_FS_DBLOCK ?
-                                  GNUNET_BLOCK_TYPE_ANY : pr->public_data.type,
-                                  (0 !=
-                                   (GSF_PRO_PRIORITY_UNLIMITED &
-                                    pr->public_data.options)) ? UINT_MAX : 1
-                                  /* queue priority */ ,
-                                  (0 !=
-                                   (GSF_PRO_PRIORITY_UNLIMITED &
-                                    pr->public_data.options)) ? UINT_MAX :
-                                  GSF_datastore_queue_size
-                                  /* max queue size */ ,
-                                  &process_local_reply, pr);
-    if (NULL != pr->qe)
-      return;                   /* we're done */
-    GNUNET_STATISTICS_update (GSF_stats,
-                              gettext_noop
-                              ("# Datastore lookups concluded (error queueing)"),
-                              1, GNUNET_NO);
-    goto check_error_and_continue;
+    start_local_query (pr,
+                       uid + 1 /* next_uid */,
+                       false /* random */);
+    return;
   }
+called_from_on_demand:
   old_rf = pr->public_data.results_found;
   memset (&prq, 0, sizeof (prq));
   prq.data = data;
@@ -1496,34 +1594,9 @@ process_local_reply (void *cls,
     GNUNET_break (0);
     GNUNET_DATASTORE_remove (GSF_dsh, key, size, data, -1, -1,
                              NULL, NULL);
-    pr->qe_start = GNUNET_TIME_absolute_get ();
-    pr->warn_task =
-        GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                      &warn_delay_task, pr);
-    pr->qe =
-        GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset - 1,
-                                  &pr->public_data.query,
-                                  pr->public_data.type ==
-                                  GNUNET_BLOCK_TYPE_FS_DBLOCK ?
-                                  GNUNET_BLOCK_TYPE_ANY : pr->public_data.type,
-                                  (0 !=
-                                   (GSF_PRO_PRIORITY_UNLIMITED &
-                                    pr->public_data.options)) ? UINT_MAX : 1
-                                  /* queue priority */ ,
-                                  (0 !=
-                                   (GSF_PRO_PRIORITY_UNLIMITED &
-                                    pr->public_data.options)) ? UINT_MAX :
-                                  GSF_datastore_queue_size
-                                  /* max queue size */ ,
-                                  &process_local_reply, pr);
-    if (NULL == pr->qe)
-    {
-      GNUNET_STATISTICS_update (GSF_stats,
-                                gettext_noop
-                                ("# Datastore lookups concluded (error queueing)"),
-                                1, GNUNET_NO);
-      goto check_error_and_continue;
-    }
+    start_local_query (pr,
+                       uid + 1 /* next_uid */,
+                       false /* random */);
     return;
   }
   prq.type = type;
@@ -1535,14 +1608,15 @@ process_local_reply (void *cls,
   prq.eo = GNUNET_BLOCK_EO_LOCAL_SKIP_CRYPTO;
   process_reply (&prq, key, pr);
   pr->local_result = prq.eval;
-  if (prq.eval == GNUNET_BLOCK_EVALUATION_OK_LAST)
+  if (GNUNET_BLOCK_EVALUATION_OK_LAST == prq.eval)
   {
     GNUNET_STATISTICS_update (GSF_stats,
                               gettext_noop
                               ("# Datastore lookups concluded (found last result)"),
                               1,
                               GNUNET_NO);
-    goto check_error_and_continue;
+    call_continuation (pr);
+    return;
   }
   if ((0 == (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options)) &&
       ((GNUNET_YES == GSF_test_get_load_too_high_ (0)) ||
@@ -1554,66 +1628,12 @@ process_local_reply (void *cls,
                               gettext_noop ("# Datastore lookups concluded (load too high)"),
                               1,
                               GNUNET_NO);
-    goto check_error_and_continue;
-  }
-  pr->qe_start = GNUNET_TIME_absolute_get ();
-  pr->warn_task =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                    &warn_delay_task,
-                                    pr);
-  pr->qe =
-      GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset++,
-                                &pr->public_data.query,
-                                pr->public_data.type ==
-                                GNUNET_BLOCK_TYPE_FS_DBLOCK ?
-                                GNUNET_BLOCK_TYPE_ANY : pr->public_data.type,
-                                (0 !=
-                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
-                                  public_data.options)) ? UINT_MAX : 1
-                                /* queue priority */ ,
-                                (0 !=
-                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
-                                  public_data.options)) ? UINT_MAX :
-                                GSF_datastore_queue_size
-                                /* max queue size */ ,
-                                &process_local_reply, pr);
-  /* check if we successfully queued another datastore request;
-   * if so, return, otherwise call our continuation (if we have
-   * any) */
-check_error_and_continue:
-  if (NULL != pr->qe)
+    call_continuation (pr);
     return;
-  if (NULL != pr->warn_task)
-  {
-    GNUNET_SCHEDULER_cancel (pr->warn_task);
-    pr->warn_task = NULL;
   }
-  if (NULL == (cont = pr->llc_cont))
-    return;                     /* no continuation */
-  pr->llc_cont = NULL;
-  if (0 != (GSF_PRO_LOCAL_ONLY & pr->public_data.options))
-  {
-    if (GNUNET_BLOCK_EVALUATION_OK_LAST != pr->local_result)
-    {
-      /* Signal that we are done and that there won't be any
-         additional results to allow client to clean up state. */
-      pr->rh (pr->rh_cls,
-               GNUNET_BLOCK_EVALUATION_OK_LAST,
-               pr,
-               UINT32_MAX,
-               GNUNET_TIME_UNIT_ZERO_ABS,
-               GNUNET_TIME_UNIT_FOREVER_ABS,
-               GNUNET_BLOCK_TYPE_ANY,
-               NULL, 0);
-    }
-    /* Finally, call our continuation to signal that we are
-       done with local processing of this request; i.e. to
-       start reading again from the client. */
-    cont (pr->llc_cont_cls, NULL, GNUNET_BLOCK_EVALUATION_OK_LAST);
-    return;
-  }
-
-  cont (pr->llc_cont_cls, pr, pr->local_result);
+  start_local_query (pr,
+                     uid + 1 /* next_uid */,
+                     false /* random */);
 }
 
 
@@ -1657,43 +1677,14 @@ GSF_local_lookup_ (struct GSF_PendingRequest *pr,
   GNUNET_assert (NULL == pr->llc_cont);
   pr->llc_cont = cont;
   pr->llc_cont_cls = cont_cls;
-  pr->qe_start = GNUNET_TIME_absolute_get ();
-  pr->warn_task =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                    &warn_delay_task,
-                                    pr);
 #if INSANE_STATISTICS
   GNUNET_STATISTICS_update (GSF_stats,
                             gettext_noop ("# Datastore lookups initiated"), 1,
                             GNUNET_NO);
 #endif
-  pr->qe =
-      GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset++,
-                                &pr->public_data.query,
-                                pr->public_data.type ==
-                                GNUNET_BLOCK_TYPE_FS_DBLOCK ?
-                                GNUNET_BLOCK_TYPE_ANY : pr->public_data.type,
-                                (0 !=
-                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
-                                  public_data.options)) ? UINT_MAX : 1
-                                /* queue priority */ ,
-                                (0 !=
-                                 (GSF_PRO_PRIORITY_UNLIMITED & pr->
-                                  public_data.options)) ? UINT_MAX :
-                                GSF_datastore_queue_size
-                                /* max queue size */ ,
-                                &process_local_reply, pr);
-  if (NULL != pr->qe)
-    return;
-  GNUNET_STATISTICS_update (GSF_stats,
-                            gettext_noop
-                            ("# Datastore lookups concluded (error queueing)"),
-                            1, GNUNET_NO);
-  GNUNET_SCHEDULER_cancel (pr->warn_task);
-  pr->warn_task = NULL;
-  pr->llc_cont = NULL;
-  if (NULL != cont)
-    cont (cont_cls, pr, pr->local_result);
+  start_local_query(pr,
+                    0 /* next_uid */,
+                    true /* random */);
 }
 
 
diff --git a/src/fs/gnunet-service-fs_put.c b/src/fs/gnunet-service-fs_put.c
index bb4cb4ecb..cd062bf2b 100644
--- a/src/fs/gnunet-service-fs_put.c
+++ b/src/fs/gnunet-service-fs_put.c
@@ -72,9 +72,14 @@ struct PutOperator
   uint64_t zero_anonymity_count_estimate;
 
   /**
-   * Current offset when iterating the database.
+   * Count of results received from the database.
    */
-  uint64_t current_offset;
+  uint64_t result_count;
+
+  /**
+   * Next UID to request when iterating the database.
+   */
+  uint64_t next_uid;
 };
 
 
@@ -177,37 +182,43 @@ delay_dht_put_task (void *cls)
  */
 static void
 process_dht_put_content (void *cls,
-                         const struct GNUNET_HashCode * key,
-                         size_t size,
+                         const struct GNUNET_HashCode * key,
+                         size_t size,
                          const void *data,
-                         enum GNUNET_BLOCK_Type type,
-                         uint32_t priority, uint32_t anonymity,
-                         struct GNUNET_TIME_Absolute expiration, uint64_t uid)
+                         enum GNUNET_BLOCK_Type type,
+                         uint32_t priority,
+                         uint32_t anonymity,
+                         struct GNUNET_TIME_Absolute expiration,
+                         uint64_t uid)
 {
   struct PutOperator *po = cls;
 
   po->dht_qe = NULL;
   if (key == NULL)
   {
-    po->zero_anonymity_count_estimate = po->current_offset - 1;
-    po->current_offset = 0;
+    po->zero_anonymity_count_estimate = po->result_count;
+    po->result_count = 0;
+    po->next_uid = 0;
     po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_task, po);
     return;
   }
+  po->result_count++;
+  po->next_uid = uid + 1;
   po->zero_anonymity_count_estimate =
-      GNUNET_MAX (po->current_offset, po->zero_anonymity_count_estimate);
+    GNUNET_MAX (po->result_count, po->zero_anonymity_count_estimate);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Retrieved block `%s' of type %u for DHT PUT\n", GNUNET_h2s (key),
               type);
   po->dht_put = GNUNET_DHT_put (GSF_dht,
                                 key,
                                 DEFAULT_PUT_REPLICATION,
-                                GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
+                                GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
                                 type,
                                 size,
                                 data,
-                                expiration,
-                                &delay_dht_put_blocks, po);
+                                expiration,
+                                &delay_dht_put_blocks,
+                                po);
 }
 
 
@@ -223,10 +234,13 @@ gather_dht_put_blocks (void *cls)
 
   po->dht_task = NULL;
   po->dht_qe =
-      GNUNET_DATASTORE_get_zero_anonymity (GSF_dsh, po->current_offset++, 0,
+      GNUNET_DATASTORE_get_zero_anonymity (GSF_dsh,
+                                           po->next_uid,
+                                           0,
                                            UINT_MAX,
                                            po->dht_put_type,
-                                           &process_dht_put_content, po);
+                                           &process_dht_put_content,
+                                           po);
   if (NULL == po->dht_qe)
     po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_task, po);
 }
diff --git a/src/fs/gnunet-unindex.c b/src/fs/gnunet-unindex.c
index 40fa13b62..c53a85fb4 100644
--- a/src/fs/gnunet-unindex.c
+++ b/src/fs/gnunet-unindex.c
@@ -30,7 +30,7 @@
 
 static int ret;
 
-static int verbose;
+static unsigned int verbose;
 
 static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
@@ -162,10 +162,10 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_set_one, &verbose},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_VERBOSE (&verbose),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/gns/gns_api.c b/src/gns/gns_api.c
index 15a59a4bc..26293f4df 100644
--- a/src/gns/gns_api.c
+++ b/src/gns/gns_api.c
@@ -391,7 +391,7 @@ GNUNET_GNS_lookup (struct GNUNET_GNS_Handle *handle,
        "Trying to lookup `%s' in GNS\n",
        name);
   nlen = strlen (name) + 1;
-  if (nlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*lr))
+  if (nlen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (*lr))
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/gns/gnunet-bcd.c b/src/gns/gnunet-bcd.c
index 21471350d..fb7ac10c1 100644
--- a/src/gns/gnunet-bcd.c
+++ b/src/gns/gnunet-bcd.c
@@ -514,10 +514,14 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'p', "port", "PORT",
-      gettext_noop ("Run HTTP serve on port PORT (default is 8888)"), 1,
-      &GNUNET_GETOPT_set_uint, &port},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('p',
+                                   "port",
+                                   "PORT",
+                                   gettext_noop ("Run HTTP serve on port PORT (default is 8888)"),
+                                   &port),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/gns/gnunet-dns2gns.c b/src/gns/gnunet-dns2gns.c
index 813ecdf8e..c9b4bde9c 100644
--- a/src/gns/gnunet-dns2gns.c
+++ b/src/gns/gnunet-dns2gns.c
@@ -776,22 +776,38 @@ int
 main (int argc,
       char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'d', "dns", "IP",
-      gettext_noop ("IP of recursive DNS resolver to use (required)"), 1,
-      &GNUNET_GETOPT_set_string, &dns_ip},
-    {'f', "fcfs", "NAME",
-      gettext_noop ("Authoritative FCFS suffix to use (optional); default: fcfs.zkey.eu"), 1,
-      &GNUNET_GETOPT_set_string, &fcfs_suffix},
-    {'s', "suffix", "SUFFIX",
-      gettext_noop ("Authoritative DNS suffix to use (optional); default: zkey.eu"), 1,
-      &GNUNET_GETOPT_set_string, &dns_suffix},
-    {'p', "port", "UDPPORT",
-      gettext_noop ("UDP port to listen on for inbound DNS requests; default: 2853"), 1,
-      &GNUNET_GETOPT_set_uint, &listen_port},
-    {'z', "zone", "PUBLICKEY",
-      gettext_noop ("Public key of the GNS zone to use (overrides default)"), 1,
-      &GNUNET_GETOPT_set_string, &gns_zone_str},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_STRING ('d',
+                                 "dns",
+                                 "IP",
+                                 gettext_noop ("IP of recursive DNS resolver to use (required)"),
+                                 &dns_ip),
+
+    GNUNET_GETOPT_OPTION_STRING ('f',
+                                 "fcfs",
+                                 "NAME",
+                                 gettext_noop ("Authoritative FCFS suffix to use (optional); default: fcfs.zkey.eu"),
+                                 &fcfs_suffix),
+
+    GNUNET_GETOPT_OPTION_STRING ('s',
+                                 "suffix",
+                                 "SUFFIX",
+                                 gettext_noop ("Authoritative DNS suffix to use (optional); default: zkey.eu"),
+                                 &dns_suffix),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('p',
+                                   "port",
+                                   "UDPPORT",
+                                   gettext_noop ("UDP port to listen on for inbound DNS requests; default: 2853"),
+                                   &listen_port),
+
+    GNUNET_GETOPT_OPTION_STRING ('z',
+                                 "zone",
+                                 "PUBLICKEY",
+                                 gettext_noop ("Public key of the GNS zone to use (overrides default)"),
+                                 &gns_zone_str), 
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/gns/gnunet-gns-proxy.c b/src/gns/gnunet-gns-proxy.c
index 6eb87a95e..35f42cdec 100644
--- a/src/gns/gnunet-gns-proxy.c
+++ b/src/gns/gnunet-gns-proxy.c
@@ -621,7 +621,7 @@ struct Socks5Request
 /**
  * The port the proxy is running on (default 7777)
  */
-static unsigned long port = GNUNET_GNS_PROXY_PORT;
+static unsigned long long port = GNUNET_GNS_PROXY_PORT;
 
 /**
  * The CA file (pem) to use for the proxy CA
@@ -865,7 +865,8 @@ check_ssl_certificate (struct Socks5Request *s5r)
   const char *name;
 
   s5r->ssl_checked = GNUNET_YES;
-  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "XXXXXX\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+              "Checking SSL certificate\n");
   if (CURLE_OK !=
       curl_easy_getinfo (s5r->curl,
                          CURLINFO_TLS_SESSION,
@@ -1882,19 +1883,22 @@ mhd_connection_cb (void *cls,
       {
         if (GNUNET_NETWORK_get_fd (s5r->sock) == sock)
         {
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Context set...\n");
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      "Context set...\n");
+          s5r->ssl_checked = GNUNET_NO;
           *con_cls = s5r;
           break;
         }
       }
-      s5r->ssl_checked = GNUNET_NO;
       break;
     case MHD_CONNECTION_NOTIFY_CLOSED:
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Connection closed... cleaning up\n");
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Connection closed... cleaning up\n");
       s5r = *con_cls;
       if (NULL == s5r)
       {
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Connection stale!\n");
+        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                    "Connection stale!\n");
         return;
       }
       cleanup_s5r (s5r);
@@ -3104,7 +3108,7 @@ run_cont ()
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Proxy listens on port %lu\n",
+              "Proxy listens on port %llu\n",
               port);
 
   /* start MHD daemon for HTTP */
@@ -3257,13 +3261,20 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'p', "port", NULL,
-      gettext_noop ("listen on specified port (default: 7777)"), 1,
-      &GNUNET_GETOPT_set_ulong, &port},
-    {'a', "authority", NULL,
-      gettext_noop ("pem file to use as CA"), 1,
-      &GNUNET_GETOPT_set_string, &cafile_opt},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_ULONG ('p',
+                                    "port",
+                                    NULL,
+                                    gettext_noop ("listen on specified port (default: 7777)"),
+                                    &port),
+
+    GNUNET_GETOPT_OPTION_STRING ('a',
+                                 "authority",
+                                 NULL,
+                                 gettext_noop ("pem file to use as CA"),
+                                 &cafile_opt),
+
     GNUNET_GETOPT_OPTION_END
   };
   static const char* page =
diff --git a/src/gns/gnunet-gns.c b/src/gns/gnunet-gns.c
index a261e008b..c85ddfe81 100644
--- a/src/gns/gnunet-gns.c
+++ b/src/gns/gnunet-gns.c
@@ -420,25 +420,43 @@ int
 main (int argc,
       char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'u', "lookup", "NAME",
-      gettext_noop ("Lookup a record for the given name"), 1,
-      &GNUNET_GETOPT_set_string, &lookup_name},
-    {'t', "type", "TYPE",
-      gettext_noop ("Specify the type of the record to lookup"), 1,
-      &GNUNET_GETOPT_set_string, &lookup_type},
-    { 'T', "timeout", "DELAY",
-      gettext_noop ("Specify timeout for the lookup"), 1,
-      &GNUNET_GETOPT_set_relative_time, &timeout },
-    {'r', "raw", NULL,
-      gettext_noop ("No unneeded output"), 0,
-      &GNUNET_GETOPT_set_one, &raw},
-    {'p', "public-key", "PKEY",
-      gettext_noop ("Specify the public key of the zone to lookup the record in"), 1,
-      &GNUNET_GETOPT_set_string, &public_key},
-    {'z', "zone", "NAME",
-      gettext_noop ("Specify the name of the ego of the zone to lookup the record in"), 1,
-      &GNUNET_GETOPT_set_string, &zone_ego_name},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_STRING ('u',
+                                 "lookup",
+                                 "NAME",
+                                 gettext_noop ("Lookup a record for the given name"),
+                                 &lookup_name),
+
+    GNUNET_GETOPT_OPTION_STRING ('t',
+                                 "type",
+                                 "TYPE",
+                                 gettext_noop ("Specify the type of the record to lookup"),
+                                 &lookup_type),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('T',
+                                            "timeout",
+                                            "DELAY",
+                                            gettext_noop ("Specify timeout for the lookup"),
+                                            &timeout),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('r',
+                                  "raw",
+                                  gettext_noop ("No unneeded output"),
+                                  &raw),
+
+    GNUNET_GETOPT_OPTION_STRING ('p',
+                                 "public-key",
+                                 "PKEY",
+                                 gettext_noop ("Specify the public key of the zone to lookup the record in"),
+                                 &public_key),
+    
+    GNUNET_GETOPT_OPTION_STRING ('z',
+                                 "zone",
+                                 "NAME",
+                                 gettext_noop ("Specify the name of the ego of the zone to lookup the record in"),
+                                 &zone_ego_name),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/gns/plugin_gnsrecord_gns.c b/src/gns/plugin_gnsrecord_gns.c
index 5faca4578..5d611e19e 100644
--- a/src/gns/plugin_gnsrecord_gns.c
+++ b/src/gns/plugin_gnsrecord_gns.c
@@ -279,9 +279,10 @@ gns_string_to_value (void *cls,
         }
         *data_size = sizeof (struct GNUNET_TUN_GnsVpnRecord) + strlen (s_serv) + 1;
         *data = vpn = GNUNET_malloc (*data_size);
-        if (GNUNET_OK != GNUNET_CRYPTO_eddsa_public_key_from_string ((char*) s_peer,
-                                                                     strlen (s_peer),
-                                                                     &vpn->peer.public_key))
+        if (GNUNET_OK !=
+            GNUNET_CRYPTO_eddsa_public_key_from_string ((char*) s_peer,
+                                                        strlen (s_peer),
+                                                        &vpn->peer.public_key))
         {
           GNUNET_free (vpn);
           *data_size = 0;
@@ -362,9 +363,14 @@ gns_string_to_value (void *cls,
         }
         *data_size = sizeof (struct GNUNET_GNSRECORD_ReverseRecord) + strlen (known_by) + 1;
         *data = rev = GNUNET_malloc (*data_size);
-        GNUNET_CRYPTO_ecdsa_public_key_from_string (pkey_str,
-                                                    strlen (pkey_str),
-                                                    &rev->pkey);
+        if (GNUNET_OK !=
+            GNUNET_CRYPTO_ecdsa_public_key_from_string (pkey_str,
+                                                        strlen (pkey_str),
+                                                        &rev->pkey))
+        {
+          GNUNET_free (rev);
+          return GNUNET_SYSERR;
+        }
         rev->expiration = expiration;
         GNUNET_memcpy (&rev[1],
                        known_by,
diff --git a/src/hello/hello.c b/src/hello/hello.c
index f9b21aa4f..27580275f 100644
--- a/src/hello/hello.c
+++ b/src/hello/hello.c
@@ -206,7 +206,7 @@ GNUNET_HELLO_create (const struct GNUNET_CRYPTO_EddsaPublicKey *public_key,
                      void *addrgen_cls,
                      int friend_only)
 {
-  char buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - 256 -
+  char buffer[GNUNET_MAX_MESSAGE_SIZE - 1 - 256 -
               sizeof (struct GNUNET_HELLO_Message)];
   size_t max;
   size_t used;
diff --git a/src/hostlist/gnunet-daemon-hostlist.c b/src/hostlist/gnunet-daemon-hostlist.c
index a83d46e07..9b4790c32 100644
--- a/src/hostlist/gnunet-daemon-hostlist.c
+++ b/src/hostlist/gnunet-daemon-hostlist.c
@@ -369,23 +369,26 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
 #if HAVE_MHD
-    {'a', "advertise", NULL,
-     gettext_noop ("advertise our hostlist to other peers"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &advertising},
+    GNUNET_GETOPT_OPTION_SET_ONE ('a',
+                                  "advertise",
+                                  gettext_noop ("advertise our hostlist to other peers"),
+                                  &advertising),
 #endif
-    {'b', "bootstrap", NULL,
-     gettext_noop
-     ("bootstrap using hostlists (it is highly recommended that you always use this option)"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &bootstrapping},
-    {'e', "enable-learning", NULL,
-     gettext_noop ("enable learning about hostlist servers from other peers"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &learning},
+    GNUNET_GETOPT_OPTION_SET_ONE ('b',
+                                  "bootstrap",
+                                  gettext_noop ("bootstrap using hostlists (it is highly recommended that you always use this option)"),
+                                  &bootstrapping),
+    GNUNET_GETOPT_OPTION_SET_ONE ('e',
+                                  "enable-learning",
+                                  gettext_noop ("enable learning about hostlist servers from other peers"),
+                                  &learning),
 #if HAVE_MHD
-    {'p', "provide-hostlist", NULL,
-     gettext_noop ("provide a hostlist server"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &provide_hostlist},
+    GNUNET_GETOPT_OPTION_SET_ONE ('p',
+                                  "provide-hostlist",
+                                  gettext_noop ("provide a hostlist server"),
+                                  &provide_hostlist),
 #endif
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/hostlist/gnunet-daemon-hostlist_client.c b/src/hostlist/gnunet-daemon-hostlist_client.c
index a973fcc28..207cc4a81 100644
--- a/src/hostlist/gnunet-daemon-hostlist_client.c
+++ b/src/hostlist/gnunet-daemon-hostlist_client.c
@@ -350,7 +350,7 @@ callback_download (void *ptr,
                    size_t nmemb,
                    void *ctx)
 {
-  static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
+  static char download_buffer[GNUNET_MAX_MESSAGE_SIZE - 1];
   const char *cbuf = ptr;
   const struct GNUNET_MessageHeader *msg;
   struct HelloOffer *ho;
@@ -373,7 +373,7 @@ callback_download (void *ptr,
   left = total;
   while ((left > 0) || (download_pos > 0))
   {
-    cpy = GNUNET_MIN (left, GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - download_pos);
+    cpy = GNUNET_MIN (left, GNUNET_MAX_MESSAGE_SIZE - 1 - download_pos);
     GNUNET_memcpy (&download_buffer[download_pos], cbuf, cpy);
     cbuf += cpy;
     download_pos += cpy;
@@ -1042,7 +1042,7 @@ download_hostlist ()
 #if 0
   CURL_EASY_SETOPT (curl, CURLOPT_VERBOSE, 1);
 #endif
-  CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_SERVER_MAX_MESSAGE_SIZE);
+  CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_MAX_MESSAGE_SIZE);
   if (0 == strncmp (current_url, "http", 4))
     CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
   CURL_EASY_SETOPT (curl, CURLOPT_CONNECTTIMEOUT, 60L);
diff --git a/src/hostlist/gnunet-daemon-hostlist_server.c b/src/hostlist/gnunet-daemon-hostlist_server.c
index 48c1a5622..3df6f1ef8 100644
--- a/src/hostlist/gnunet-daemon-hostlist_server.c
+++ b/src/hostlist/gnunet-daemon-hostlist_server.c
@@ -489,7 +489,7 @@ connect_handler (void *cls,
     return NULL;
   size = strlen (hostlist_uri) + 1;
   if (size + sizeof (struct GNUNET_MessageHeader) >=
-      GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/identity-provider/gnunet-identity-token.c b/src/identity-provider/gnunet-identity-token.c
index 4bb3292be..906899ea7 100644
--- a/src/identity-provider/gnunet-identity-token.c
+++ b/src/identity-provider/gnunet-identity-token.c
@@ -158,13 +158,18 @@ run (void *cls,
 int
 main(int argc, char *const argv[])
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'t', "token", NULL,
-      gettext_noop ("GNUid token"), 1,
-      &GNUNET_GETOPT_set_string, &token},
-    {'p', "print", NULL,
-      gettext_noop ("Print token contents"), 0,
-      &GNUNET_GETOPT_set_one, &print_token},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_STRING ('t',
+                                 "token",
+                                 NULL,
+                                 gettext_noop ("GNUid token"),
+                                 &token),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('p',
+                                  "print",
+                                  gettext_noop ("Print token contents"),
+                                  &print_token),
 
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/identity-provider/identity_provider_api.c b/src/identity-provider/identity_provider_api.c
index 32c3cb703..845d1f753 100644
--- a/src/identity-provider/identity_provider_api.c
+++ b/src/identity-provider/identity_provider_api.c
@@ -440,7 +440,7 @@ GNUNET_IDENTITY_PROVIDER_issue_token (struct GNUNET_IDENTITY_PROVIDER_Handle *id
   size_t slen;
 
   slen = strlen (scopes) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct IssueMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct IssueMessage))
   {
     GNUNET_break (0);
     return NULL;
@@ -493,7 +493,7 @@ GNUNET_IDENTITY_PROVIDER_exchange_ticket (struct GNUNET_IDENTITY_PROVIDER_Handle
   ticket_str = GNUNET_IDENTITY_PROVIDER_ticket_to_string (ticket);
 
   slen = strlen (ticket_str) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct ExchangeMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct ExchangeMessage))
   {
     GNUNET_free (ticket_str);
     GNUNET_break (0);
diff --git a/src/identity/gnunet-identity.c b/src/identity/gnunet-identity.c
index 8c8485249..5c457ef5d 100644
--- a/src/identity/gnunet-identity.c
+++ b/src/identity/gnunet-identity.c
@@ -349,25 +349,41 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'C', "create", "NAME",
-     gettext_noop ("create ego NAME"),
-     1, &GNUNET_GETOPT_set_string, &create_ego},
-    {'D', "delete", "NAME",
-     gettext_noop ("delete ego NAME "),
-     1, &GNUNET_GETOPT_set_string, &delete_ego},
-    {'d', "display", NULL,
-     gettext_noop ("display all egos"),
-     0, &GNUNET_GETOPT_set_one, &list},
-    {'e', "ego", "NAME",
-     gettext_noop ("set default identity to EGO for a subsystem SUBSYSTEM (use together with -s)"),
-     1, &GNUNET_GETOPT_set_string, &set_ego},
-    {'m', "monitor", NULL,
-     gettext_noop ("run in monitor mode egos"),
-     0, &GNUNET_GETOPT_set_one, &monitor},
-    {'s', "set", "SUBSYSTEM",
-     gettext_noop ("set default identity to EGO for a subsystem SUBSYSTEM (use together with -e)"),
-     1, &GNUNET_GETOPT_set_string, &set_subsystem},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_STRING ('C',
+                                 "create",
+                                 "NAME",
+                                 gettext_noop ("create ego NAME"),
+                                 &create_ego),
+
+    GNUNET_GETOPT_OPTION_STRING ('D',
+                                 "delete",
+                                 "NAME",
+                                 gettext_noop ("delete ego NAME "),
+                                 &delete_ego),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('d',
+                                  "display",
+                                  gettext_noop ("display all egos"),
+                                  &list),
+    
+    GNUNET_GETOPT_OPTION_STRING ('e',
+                                 "ego",
+                                 "NAME",
+                                 gettext_noop ("set default identity to EGO for a subsystem SUBSYSTEM (use together with -s)"),
+                                 &set_ego),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('m',
+                                  "monitor",
+                                  gettext_noop ("run in monitor mode egos"),
+                                  &monitor),
+
+    GNUNET_GETOPT_OPTION_STRING ('s',
+                                 "set",
+                                 "SUBSYSTEM",
+                                 gettext_noop ("set default identity to EGO for a subsystem SUBSYSTEM (use together with -e)"),
+                                 &set_subsystem),
+
     GNUNET_GETOPT_OPTION_END
   };
   int res;
diff --git a/src/identity/identity_api.c b/src/identity/identity_api.c
index 905b3fd8b..1601ae2fd 100644
--- a/src/identity/identity_api.c
+++ b/src/identity/identity_api.c
@@ -671,7 +671,7 @@ GNUNET_IDENTITY_get (struct GNUNET_IDENTITY_Handle *h,
   if (NULL == h->mq)
     return NULL;
   slen = strlen (service_name) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct GetDefaultMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct GetDefaultMessage))
   {
     GNUNET_break (0);
     return NULL;
@@ -722,7 +722,7 @@ GNUNET_IDENTITY_set (struct GNUNET_IDENTITY_Handle *h,
   if (NULL == h->mq)
     return NULL;
   slen = strlen (service_name) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct SetDefaultMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct SetDefaultMessage))
   {
     GNUNET_break (0);
     return NULL;
@@ -773,7 +773,7 @@ GNUNET_IDENTITY_create (struct GNUNET_IDENTITY_Handle *h,
   if (NULL == h->mq)
     return NULL;
   slen = strlen (name) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct CreateRequestMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct CreateRequestMessage))
   {
     GNUNET_break (0);
     return NULL;
@@ -830,9 +830,9 @@ GNUNET_IDENTITY_rename (struct GNUNET_IDENTITY_Handle *h,
     return NULL;
   slen_old = strlen (old_name) + 1;
   slen_new = strlen (new_name) + 1;
-  if ( (slen_old >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-       (slen_new >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-       (slen_old + slen_new >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct RenameMessage)) )
+  if ( (slen_old >= GNUNET_MAX_MESSAGE_SIZE) ||
+       (slen_new >= GNUNET_MAX_MESSAGE_SIZE) ||
+       (slen_old + slen_new >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct RenameMessage)) )
   {
     GNUNET_break (0);
     return NULL;
@@ -885,7 +885,7 @@ GNUNET_IDENTITY_delete (struct GNUNET_IDENTITY_Handle *h,
   if (NULL == h->mq)
     return NULL;
   slen = strlen (name) + 1;
-  if (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct DeleteMessage))
+  if (slen >= GNUNET_MAX_MESSAGE_SIZE - sizeof (struct DeleteMessage))
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/include/Makefile.am b/src/include/Makefile.am
index 4fb2577fd..b745da125 100644
--- a/src/include/Makefile.am
+++ b/src/include/Makefile.am
@@ -38,7 +38,6 @@ gnunetinclude_HEADERS = \
   gnunet_common.h \
   gnunet_constants.h \
   gnunet_configuration_lib.h \
-  gnunet_connection_lib.h \
   gnunet_consensus_service.h \
   gnunet_container_lib.h \
   gnunet_conversation_service.h \
@@ -107,7 +106,6 @@ gnunetinclude_HEADERS = \
   gnunet_scalarproduct_service.h \
   gnunet_scheduler_lib.h \
   gnunet_secretsharing_service.h \
-  gnunet_server_lib.h \
   gnunet_service_lib.h \
   gnunet_set_service.h \
   gnunet_signal_lib.h \
diff --git a/src/include/gnunet_arm_service.h b/src/include/gnunet_arm_service.h
index 7fb14d3ac..f8d71bd8b 100644
--- a/src/include/gnunet_arm_service.h
+++ b/src/include/gnunet_arm_service.h
@@ -62,25 +62,9 @@ enum GNUNET_ARM_RequestStatus
   GNUNET_ARM_REQUEST_SENT_OK = 0,
 
   /**
-   * Misconfiguration (can't connect to the ARM service).
-   */
-  GNUNET_ARM_REQUEST_CONFIGURATION_ERROR = 1,
-
-  /**
    * We disconnected from ARM, and request was not sent.
    */
-  GNUNET_ARM_REQUEST_DISCONNECTED = 2,
-
-  /**
-   * ARM API is busy (probably trying to connect to ARM),
-   * and request was not sent. Try again later.
-   */
-  GNUNET_ARM_REQUEST_BUSY = 3,
-
-  /**
-   * Request time ran out before we had a chance to send it.
-   */
-  GNUNET_ARM_REQUEST_TIMEOUT = 5
+  GNUNET_ARM_REQUEST_DISCONNECTED = 2
 
 };
 
diff --git a/src/include/gnunet_bandwidth_lib.h b/src/include/gnunet_bandwidth_lib.h
index 178ddaaac..967d50dea 100644
--- a/src/include/gnunet_bandwidth_lib.h
+++ b/src/include/gnunet_bandwidth_lib.h
@@ -133,7 +133,7 @@ struct GNUNET_BANDWIDTH_Tracker
   /**
    * Maximum number of seconds over which bandwidth may "accumulate".
    * Note that additionally, we also always allow at least
-   * #GNUNET_SERVER_MAX_MESSAGE_SIZE to accumulate.
+   * #GNUNET_MAX_MESSAGE_SIZE to accumulate.
    */
   uint32_t max_carry_s__;
 };
@@ -214,10 +214,10 @@ GNUNET_BANDWIDTH_value_max (struct GNUNET_BANDWIDTH_Value32NBO b1,
 /**
  * Initialize bandwidth tracker.  Note that in addition to the
  * 'max_carry_s' limit, we also always allow at least
- * #GNUNET_SERVER_MAX_MESSAGE_SIZE to accumulate.  So if the
+ * #GNUNET_MAX_MESSAGE_SIZE to accumulate.  So if the
  * bytes-per-second limit is so small that within 'max_carry_s' not
- * even #GNUNET_SERVER_MAX_MESSAGE_SIZE is allowed to accumulate, it is
- * ignored and replaced by #GNUNET_SERVER_MAX_MESSAGE_SIZE (which is in
+ * even #GNUNET_MAX_MESSAGE_SIZE is allowed to accumulate, it is
+ * ignored and replaced by #GNUNET_MAX_MESSAGE_SIZE (which is in
  * bytes).
  *
  * @param av tracker to initialize
@@ -238,10 +238,10 @@ GNUNET_BANDWIDTH_tracker_init (struct GNUNET_BANDWIDTH_Tracker *av,
 /**
  * Initialize bandwidth tracker.  Note that in addition to the
  * 'max_carry_s' limit, we also always allow at least
- * GNUNET_SERVER_MAX_MESSAGE_SIZE to accumulate.  So if the
+ * GNUNET_MAX_MESSAGE_SIZE to accumulate.  So if the
  * bytes-per-second limit is so small that within 'max_carry_s' not
- * even GNUNET_SERVER_MAX_MESSAGE_SIZE is allowed to accumulate, it is
- * ignored and replaced by GNUNET_SERVER_MAX_MESSAGE_SIZE (which is in
+ * even GNUNET_MAX_MESSAGE_SIZE is allowed to accumulate, it is
+ * ignored and replaced by GNUNET_MAX_MESSAGE_SIZE (which is in
  * bytes).
  *
  * @param av tracker to initialize
diff --git a/src/include/gnunet_cadet_service.h b/src/include/gnunet_cadet_service.h
index fd838df8d..f76f17a51 100644
--- a/src/include/gnunet_cadet_service.h
+++ b/src/include/gnunet_cadet_service.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009-2014 GNUnet e.V.
+     Copyright (C) 2009-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -49,7 +49,7 @@ extern "C"
 /**
  * Version number of GNUnet-cadet API.
  */
-#define GNUNET_CADET_VERSION 0x00000004
+#define GNUNET_CADET_VERSION 0x00000005
 
 
 /**
@@ -69,6 +69,33 @@ struct GNUNET_CADET_Port;
 
 
 /**
+ * Hash uniquely identifying a connection below a tunnel.
+ */
+struct GNUNET_CADET_ConnectionTunnelIdentifier
+{
+  struct GNUNET_ShortHashCode connection_of_tunnel;
+};
+
+
+/**
+ * Number identifying a CADET channel within a tunnel.
+ */
+struct GNUNET_CADET_ChannelTunnelNumber
+{
+  /**
+   * Which number does this channel have that uniquely identfies
+   * it within its tunnel, in network byte order.
+   *
+   * Given two peers, both may initiate channels over the same tunnel.
+   * The @e cn must be greater or equal to 0x80000000 (high-bit set)
+   * for tunnels initiated with the peer that has the larger peer
+   * identity as compared using #GNUNET_CRYPTO_cmp_peer_identity().
+   */
+  uint32_t cn GNUNET_PACKED;
+};
+
+
+/**
  * Channel options.  Second line indicates filed in the
  * CadetChannelInfo union carrying the answer.
  */
@@ -108,118 +135,67 @@ enum GNUNET_CADET_ChannelOption
 
 
 /**
- * Functions with this signature are called whenever a message is
- * received.
- *
- * Each time the function must call #GNUNET_CADET_receive_done on the channel
- * in order to receive the next message. This doesn't need to be immediate:
- * can be delayed if some processing is done on the message.
+ * Method called whenever a peer connects to a port in MQ-based CADET.
  *
- * @param cls Closure (set from #GNUNET_CADET_connect).
- * @param channel Connection to the other end.
- * @param channel_ctx Place to store local state associated with the channel.
- * @param message The actual message.
- * @return #GNUNET_OK to keep the channel open,
- *         #GNUNET_SYSERR to close it (signal serious error).
- */
-typedef int
-(*GNUNET_CADET_MessageCallback) (void *cls,
-                                 struct GNUNET_CADET_Channel *channel,
-                                 void **channel_ctx,
-                                 const struct GNUNET_MessageHeader *message);
-
-
-/**
- * Message handler.  Each struct specifies how to handle on particular
- * type of message received.
+ * @param cls Closure from #GNUNET_CADET_open_port.
+ * @param channel New handle to the channel.
+ * @param source Peer that started this channel.
+ * @return Closure for the incoming @a channel. It's given to:
+ *         - The #GNUNET_CADET_DisconnectEventHandler (given to
+ *           #GNUNET_CADET_open_port) when the channel dies.
+ *         - Each the #GNUNET_MQ_MessageCallback handlers for each message
+ *           received on the @a channel.
  */
-struct GNUNET_CADET_MessageHandler
-{
-  /**
-   * Function to call for messages of type @e type.
-   */
-  GNUNET_CADET_MessageCallback callback;
-
-  /**
-   * Type of the message this handler covers.
-   */
-  uint16_t type;
-
-  /**
-   * Expected size of messages of this type.  Use 0 for variable-size.
-   * If non-zero, messages of the given type will be discarded if they
-   * do not have the right size.
-   */
-  uint16_t expected_size;
-};
+typedef void *
+(*GNUNET_CADET_ConnectEventHandler) (void *cls,
+                                     struct GNUNET_CADET_Channel *channel,
+                                     const struct GNUNET_PeerIdentity *source);
 
 
 /**
- * Method called whenever another peer has added us to a channel
- * the other peer initiated.
- * Only called (once) upon reception of data with a message type which was
- * subscribed to in #GNUNET_CADET_connect.
- *
- * A call to #GNUNET_CADET_channel_destroy causes te channel to be ignored. In
- * this case the handler MUST return NULL.
+ * Function called whenever an MQ-channel is destroyed, even if the destruction
+ * was requested by #GNUNET_CADET_channel_destroy.
+ * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
  *
- * @param cls closure
- * @param channel new handle to the channel
- * @param initiator peer that started the channel
- * @param port Port this channel is for.
- * @param options CadetOption flag field, with all active option bits set to 1.
+ * It should clean up any associated state, including cancelling any pending
+ * transmission on this channel.
  *
- * @return initial channel context for the channel
- *         (can be NULL -- that's not an error)
+ * @param cls Channel closure.
+ * @param channel Connection to the other end (henceforth invalid).
  */
-typedef void *
-(GNUNET_CADET_InboundChannelNotificationHandler) (void *cls,
-                                                  struct GNUNET_CADET_Channel *channel,
-                                                  const struct GNUNET_PeerIdentity *initiator,
-                                                  const struct GNUNET_HashCode *port,
-                                                  enum GNUNET_CADET_ChannelOption options);
+typedef void
+(*GNUNET_CADET_DisconnectEventHandler) (void *cls,
+                                        const struct GNUNET_CADET_Channel *channel);
 
 
 /**
- * Function called whenever a channel is destroyed.  Should clean up
- * any associated state, including cancelling any pending transmission on this
- * channel.
+ * Function called whenever an MQ-channel's transmission window size changes.
  *
- * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
+ * The first callback in an outgoing channel will be with a non-zero value
+ * and will mean the channel is connected to the destination.
+ *
+ * For an incoming channel it will be called immediately after the
+ * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
  *
- * @param cls closure (set from #GNUNET_CADET_connect)
- * @param channel connection to the other end (henceforth invalid)
- * @param channel_ctx place where local state associated
- *                   with the channel is stored
+ * @param cls Channel closure.
+ * @param channel Connection to the other end --- FIXME: drop?
+ * @param window_size New window size. If the is more messages than buffer size
+ *                    this value will be negative. -- FIXME: make unsigned, we never call negative?
  */
 typedef void
-(GNUNET_CADET_ChannelEndHandler) (void *cls,
-                                  const struct GNUNET_CADET_Channel *channel,
-                                  void *channel_ctx);
+(*GNUNET_CADET_WindowSizeEventHandler) (void *cls,
+                                        const struct GNUNET_CADET_Channel *channel,
+                                        int window_size);
 
 
 /**
- * Connect to the cadet service.
+ * Connect to the MQ-based cadet service.
  *
  * @param cfg Configuration to use.
- * @param cls Closure for the various callbacks that follow (including
- *            handlers in the handlers array).
- * @param cleaner Function called when a channel is destroyed.
- *                It is called immediately if #GNUNET_CADET_channel_destroy
- *                is called on the channel.
- * @param handlers Callbacks for messages we care about, NULL-terminated. Each
- *                 one must call #GNUNET_CADET_receive_done on the channel to
- *                 receive the next message.  Messages of a type that is not
- *                 in the handlers array are ignored if received.
- *
- * @return handle to the cadet service NULL on error
- *         (in this case, init is never called)
+ * @return Handle to the cadet service NULL on error.
  */
 struct GNUNET_CADET_Handle *
-GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      void *cls,
-                      GNUNET_CADET_ChannelEndHandler cleaner,
-                      const struct GNUNET_CADET_MessageHandler *handlers);
+GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg);
 
 
 /**
@@ -233,21 +209,29 @@ GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
 void
 GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle);
 
+
 /**
- * Open a port to receive incomming channels.
+ * Open a port to receive incomming MQ-based channels.
  *
  * @param h CADET handle.
- * @param port Hash representing the port number.
- * @param new_channel Function called when an channel is received.
- * @param new_channel_cls Closure for @a new_channel.
- *
+ * @param port Hash identifying the port.
+ * @param connects Function called when an incoming channel is connected.
+ * @param connects_cls Closure for the @a connects handler.
+ * @param window_changes Function called when the transmit window size changes.
+ *                       Can be NULL.
+ * @param disconnects Function called when a channel is disconnected.
+ * @param handlers Callbacks for messages we care about, NULL-terminated.
  * @return Port handle.
  */
 struct GNUNET_CADET_Port *
 GNUNET_CADET_open_port (struct GNUNET_CADET_Handle *h,
-                        const struct GNUNET_HashCode *port,
-                        GNUNET_CADET_InboundChannelNotificationHandler new_channel,
-                        void *new_channel_cls);
+                        const struct GNUNET_HashCode *port,
+                        GNUNET_CADET_ConnectEventHandler connects,
+                        void *connects_cls,
+                        GNUNET_CADET_WindowSizeEventHandler window_changes,
+                        GNUNET_CADET_DisconnectEventHandler disconnects,
+                        const struct GNUNET_MQ_MessageHandler *handlers);
+
 
 /**
  * Close a port opened with @a GNUNET_CADET_open_port.
@@ -258,27 +242,38 @@ GNUNET_CADET_open_port (struct GNUNET_CADET_Handle *h,
 void
 GNUNET_CADET_close_port (struct GNUNET_CADET_Port *p);
 
+
 /**
  * Create a new channel towards a remote peer.
  *
  * If the destination port is not open by any peer or the destination peer
- * does not accept the channel, #GNUNET_CADET_ChannelEndHandler will be called
+ * does not accept the channel, @a disconnects will be called
  * for this channel.
  *
- * @param h cadet handle
- * @param channel_ctx client's channel context to associate with the channel
- * @param peer peer identity the channel should go to
- * @param port Port hash (port number).
+ * @param h CADET handle.
+ * @param channel_cls Closure for the channel. It's given to:
+ *                    - The management handler @a window_changes.
+ *                    - The disconnect handler @a disconnects
+ *                    - Each message type callback in @a handlers
+ * @param destination Peer identity the channel should go to.
+ * @param port Identification of the destination port.
  * @param options CadetOption flag field, with all desired option bits set to 1.
- *
- * @return handle to the channel
+ * @param window_changes Function called when the transmit window size changes.
+ *                       Can be NULL if this data is of no interest.
+ * TODO                  Not yet implemented.
+ * @param disconnects Function called when the channel is disconnected.
+ * @param handlers Callbacks for messages we care about, NULL-terminated.
+ * @return Handle to the channel.
  */
 struct GNUNET_CADET_Channel *
 GNUNET_CADET_channel_create (struct GNUNET_CADET_Handle *h,
-                            void *channel_ctx,
-                            const struct GNUNET_PeerIdentity *peer,
-                            const struct GNUNET_HashCode *port,
-                            enum GNUNET_CADET_ChannelOption options);
+                             void *channel_cls,
+                             const struct GNUNET_PeerIdentity *destination,
+                             const struct GNUNET_HashCode *port,
+                             enum GNUNET_CADET_ChannelOption options,
+                             GNUNET_CADET_WindowSizeEventHandler window_changes,
+                             GNUNET_CADET_DisconnectEventHandler disconnects,
+                             const struct GNUNET_MQ_MessageHandler *handlers);
 
 
 /**
@@ -295,6 +290,52 @@ GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel);
 
 
 /**
+ * Obtain the message queue for a connected channel.
+ *
+ * @param channel The channel handle from which to get the MQ.
+ * @return The message queue of the channel.
+ */
+struct GNUNET_MQ_Handle *
+GNUNET_CADET_get_mq (const struct GNUNET_CADET_Channel *channel);
+
+
+/**
+ * Indicate readiness to receive the next message on a channel.
+ *
+ * Should only be called once per handler called.
+ *
+ * @param channel Channel that will be allowed to call another handler.
+ */
+void
+GNUNET_CADET_receive_done (struct GNUNET_CADET_Channel *channel);
+
+
+/******************************************************************************/
+/********************       MONITORING /DEBUG API     *************************/
+/******************************************************************************/
+/* The following calls are not useful for normal CADET operation, but for      */
+/* debug and monitoring of the cadet state. They can be safely ignored.        */
+/* The API can change at any point without notice.                            */
+/* Please contact the developer if you consider any of this calls useful for  */
+/* normal cadet applications.                                                  */
+/******************************************************************************/
+
+
+/**
+ * Transitional function to convert an unsigned int port to a hash value.
+ * WARNING: local static value returned, NOT reentrant!
+ * WARNING: do not use this function for new code!
+ *
+ * @param port Numerical port (unsigned int format).
+ *
+ * @return A GNUNET_HashCode usable for the new CADET API.
+ */
+const struct GNUNET_HashCode *
+GC_u2h (uint32_t port);
+
+
+
+/**
  * Struct to retrieve info about a channel.
  */
 union GNUNET_CADET_ChannelInfo
@@ -327,76 +368,6 @@ GNUNET_CADET_channel_get_info (struct GNUNET_CADET_Channel *channel,
 
 
 /**
- * Handle for a transmission request.
- */
-struct GNUNET_CADET_TransmitHandle;
-
-
-/**
- * Ask the cadet to call @a notify once it is ready to transmit the
- * given number of bytes to the specified channel.
- * Only one call can be active at any time, to issue another request,
- * wait for the callback or cancel the current request.
- *
- * @param channel channel to use for transmission
- * @param cork is corking allowed for this transmission?
- * @param maxdelay how long can the message wait?
- * @param notify_size how many bytes of buffer space does notify want?
- * @param notify function to call when buffer space is available;
- *        will be called with NULL on timeout or if the overall queue
- *        for this peer is larger than queue_size and this is currently
- *        the message with the lowest priority
- * @param notify_cls closure for @a notify
- * @return non-NULL if the notify callback was queued,
- *         NULL if we can not even queue the request (insufficient
- *         memory); if NULL is returned, @a notify will NOT be called.
- */
-struct GNUNET_CADET_TransmitHandle *
-GNUNET_CADET_notify_transmit_ready (struct GNUNET_CADET_Channel *channel,
-                                   int cork,
-                                   struct GNUNET_TIME_Relative maxdelay,
-                                   size_t notify_size,
-                                   GNUNET_CONNECTION_TransmitReadyNotify notify,
-                                   void *notify_cls);
-
-
-/**
- * Cancel the specified transmission-ready notification.
- *
- * #DEPRECATED
- * Since soon we will send immediately with mq (via request_data),
- * there will be time or need to cancel a "pending" transmission.
- *
- * @param th handle that was returned by "notify_transmit_ready".
- */
-void
-GNUNET_CADET_notify_transmit_ready_cancel (struct GNUNET_CADET_TransmitHandle *th);
-
-
-/**
- * Indicate readiness to receive the next message on a channel.
- *
- * Should only be called once per handler called.
- *
- * @param channel Channel that will be allowed to call another handler.
- */
-void
-GNUNET_CADET_receive_done (struct GNUNET_CADET_Channel *channel);
-
-
-
-/******************************************************************************/
-/********************       MONITORING /DEBUG API     *************************/
-/******************************************************************************/
-/* The following calls are not useful for normal CADET operation, but for      */
-/* debug and monitoring of the cadet state. They can be safely ignored.        */
-/* The API can change at any point without notice.                            */
-/* Please contact the developer if you consider any of this calls useful for  */
-/* normal cadet applications.                                                  */
-/******************************************************************************/
-
-
-/**
  * Method called to retrieve information about a specific channel the cadet peer
  * is aware of, including all transit nodes.
  *
@@ -482,33 +453,6 @@ typedef void
 
 
 /**
- * Hash uniquely identifying a connection below a tunnel.
- */
-struct GNUNET_CADET_ConnectionTunnelIdentifier
-{
-  struct GNUNET_ShortHashCode connection_of_tunnel;
-};
-
-
-/**
- * Number identifying a CADET channel within a tunnel.
- */
-struct GNUNET_CADET_ChannelTunnelNumber
-{
-  /**
-   * Which number does this channel have that uniquely identfies
-   * it within its tunnel, in network byte order.
-   *
-   * Given two peers, both may initiate channels over the same tunnel.
-   * The @e cn must be greater or equal to 0x80000000 (high-bit set)
-   * for tunnels initiated with the peer that has the larger peer
-   * identity as compared using #GNUNET_CRYPTO_cmp_peer_identity().
-   */
-  uint32_t cn GNUNET_PACKED;
-};
-
-
-/**
  * Method called to retrieve information about a specific tunnel the cadet peer
  * has established, o`r is trying to establish.
  *
@@ -667,169 +611,6 @@ GNUNET_CADET_get_tunnel (struct GNUNET_CADET_Handle *h,
                          void *callback_cls);
 
 
-/**
- * Create a message queue for a cadet channel.
- * The message queue can only be used to transmit messages,
- * not to receive them.
- *
- * @param channel the channel to create the message qeue for
- * @return a message queue to messages over the channel
- */
-struct GNUNET_MQ_Handle *
-GNUNET_CADET_mq_create (struct GNUNET_CADET_Channel *channel);
-
-
-/**
- * Transitional function to convert an unsigned int port to a hash value.
- * WARNING: local static value returned, NOT reentrant!
- * WARNING: do not use this function for new code!
- *
- * @param port Numerical port (unsigned int format).
- *
- * @return A GNUNET_HashCode usable for the new CADET API.
- */
-const struct GNUNET_HashCode *
-GC_u2h (uint32_t port);
-
-
-/******************************************************************************/
-/******************************* MQ-BASED API *********************************/
-/******************************************************************************/
-
-/**
- * Method called whenever a peer connects to a port in MQ-based CADET.
- *
- * @param cls Closure from #GNUNET_CADET_open_porT.
- * @param channel New handle to the channel.
- * @param source Peer that started this channel.
- * @return Closure for the incoming @a channel. It's given to:
- *         - The #GNUNET_CADET_DisconnectEventHandler (given to
- *           #GNUNET_CADET_open_porT) when the channel dies.
- *         - Each the #GNUNET_MQ_MessageCallback handlers for each message
- *           received on the @a channel.
- */
-typedef void *
-(*GNUNET_CADET_ConnectEventHandler) (void *cls,
-                                     struct GNUNET_CADET_Channel *channel,
-                                     const struct GNUNET_PeerIdentity *source);
-
-
-/**
- * Function called whenever an MQ-channel is destroyed, even if the destruction
- * was requested by #GNUNET_CADET_channel_destroy.
- * It must NOT call #GNUNET_CADET_channel_destroy on the channel.
- *
- * It should clean up any associated state, including cancelling any pending
- * transmission on this channel.
- *
- * @param cls Channel closure.
- * @param channel Connection to the other end (henceforth invalid).
- */
-typedef void
-(*GNUNET_CADET_DisconnectEventHandler) (void *cls,
-                                        const struct GNUNET_CADET_Channel *channel);
-
-
-/**
- * Function called whenever an MQ-channel's transmission window size changes.
- *
- * The first callback in an outgoing channel will be with a non-zero value
- * and will mean the channel is connected to the destination.
- *
- * For an incoming channel it will be called immediately after the
- * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
- *
- * @param cls Channel closure.
- * @param channel Connection to the other end --- FIXME: drop?
- * @param window_size New window size. If the is more messages than buffer size
- *                    this value will be negative. -- FIXME: make unsigned, we never call negative?
- */
-typedef void
-(*GNUNET_CADET_WindowSizeEventHandler) (void *cls,
-                                        const struct GNUNET_CADET_Channel *channel,
-                                        int window_size);
-
-
-/**
- * Connect to the MQ-based cadet service.
- *
- * @param cfg Configuration to use.
- * @return Handle to the cadet service NULL on error.
- */
-struct GNUNET_CADET_Handle *
-GNUNET_CADET_connecT (const struct GNUNET_CONFIGURATION_Handle *cfg);
-
-
-/**
- * Open a port to receive incomming MQ-based channels.
- *
- * @param h CADET handle.
- * @param port Hash identifying the port.
- * @param connects Function called when an incoming channel is connected.
- * @param connects_cls Closure for the @a connects handler.
- * @param window_changes Function called when the transmit window size changes.
- *                       Can be NULL.
- * @param disconnects Function called when a channel is disconnected.
- * @param handlers Callbacks for messages we care about, NULL-terminated.
- * @return Port handle.
- */
-struct GNUNET_CADET_Port *
-GNUNET_CADET_open_porT (struct GNUNET_CADET_Handle *h,
-                        const struct GNUNET_HashCode *port,
-                        GNUNET_CADET_ConnectEventHandler connects,
-                        void *connects_cls,
-                        GNUNET_CADET_WindowSizeEventHandler window_changes,
-                        GNUNET_CADET_DisconnectEventHandler disconnects,
-                        const struct GNUNET_MQ_MessageHandler *handlers);
-
-/**
- * Create a new channel towards a remote peer.
- *
- * If the destination port is not open by any peer or the destination peer
- * does not accept the channel, #GNUNET_CADET_ChannelEndHandler will be called
- * for this channel.
- *
- * @param h CADET handle.
- * @param channel_cls Closure for the channel. It's given to:
- *                    - The management handler @a window_changes.
- *                    - The disconnect handler @a disconnects
- *                    - Each message type callback in @a handlers
- * @param destination Peer identity the channel should go to.
- * @param port Identification of the destination port.
- * @param options CadetOption flag field, with all desired option bits set to 1.
- * @param window_changes Function called when the transmit window size changes.
- *                       Can be NULL if this data is of no interest.
- * TODO                  Not yet implemented.
- * @param disconnects Function called when the channel is disconnected.
- * @param handlers Callbacks for messages we care about, NULL-terminated.
- * @return Handle to the channel.
- */
-struct GNUNET_CADET_Channel *
-GNUNET_CADET_channel_creatE (struct GNUNET_CADET_Handle *h,
-                             void *channel_cls,
-                             const struct GNUNET_PeerIdentity *destination,
-                             const struct GNUNET_HashCode *port,
-                             enum GNUNET_CADET_ChannelOption options,
-                             GNUNET_CADET_WindowSizeEventHandler window_changes,
-                             GNUNET_CADET_DisconnectEventHandler disconnects,
-                             const struct GNUNET_MQ_MessageHandler *handlers);
-
-
-/**
- * Obtain the message queue for a connected channel.
- *
- * @param channel The channel handle from which to get the MQ.
- * @return The message queue of the channel.
- */
-struct GNUNET_MQ_Handle *
-GNUNET_CADET_get_mq (const struct GNUNET_CADET_Channel *channel);
-
-
-/******************************************************************************/
-/******************************* MQ-BASED API *********************************/
-/******************************************************************************/
-
-
 
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
diff --git a/src/include/gnunet_configuration_lib.h b/src/include/gnunet_configuration_lib.h
index 746dea61f..c1537e4f8 100644
--- a/src/include/gnunet_configuration_lib.h
+++ b/src/include/gnunet_configuration_lib.h
@@ -281,8 +281,8 @@ GNUNET_CONFIGURATION_get_value_number (const struct GNUNET_CONFIGURATION_Handle
  * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
  */
 int
-GNUNET_CONFIGURATION_get_value_float  (const struct GNUNET_CONFIGURATION_Handle
-                                       *cfg, const char *section,
+GNUNET_CONFIGURATION_get_value_float  (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                                       const char *section,
                                        const char *option,
                                        float *number);
 
@@ -454,9 +454,6 @@ GNUNET_CONFIGURATION_get_data (const struct GNUNET_CONFIGURATION_Handle *cfg,
                                size_t buf_size);
 
 
-
-
-
 /**
  * Expand an expression of the form "$FOO/BAR" to "DIRECTORY/BAR"
  * where either in the "PATHS" section or the environtment "FOO" is
diff --git a/src/include/gnunet_connection_lib.h b/src/include/gnunet_connection_lib.h
deleted file mode 100644
index e9dd95d1b..000000000
--- a/src/include/gnunet_connection_lib.h
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @author Christian Grothoff
- *
- * @file include/gnunet_connection_lib.h
- * Basic, low-level TCP networking interface
- *
- * @defgroup connection  Connection library
- * Basic, low-level TCP networking interface
- * @{
- */
-#ifndef GNUNET_CONNECTION_LIB_H
-#define GNUNET_CONNECTION_LIB_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "gnunet_network_lib.h"
-#include "gnunet_scheduler_lib.h"
-#include "gnunet_time_lib.h"
-
-/**
- * Timeout we use on TCP connect before trying another
- * result from the DNS resolver.  Actual value used
- * is this value divided by the number of address families.
- * Default is 5s.
- */
-#define GNUNET_CONNECTION_CONNECT_RETRY_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
-
-/**
- * @brief handle for a network connection
- */
-struct GNUNET_CONNECTION_Handle;
-
-
-/**
- * Credentials for UNIX domain sockets.
- */
-struct GNUNET_CONNECTION_Credentials
-{
-  /**
-   * UID of the other end of the connection.
-   */
-  uid_t uid;
-
-  /**
-   * GID of the other end of the connection.
-   */
-  gid_t gid;
-};
-
-
-/**
- * Function to call for access control checks.
- *
- * @param cls closure
- * @param ucred credentials, if available, otherwise NULL
- * @param addr address
- * @param addrlen length of address
- * @return GNUNET_YES to allow, GNUNET_NO to deny, GNUNET_SYSERR
- *   for unknown address family (will be denied).
- */
-typedef int (*GNUNET_CONNECTION_AccessCheck) (void *cls,
-                                              const struct
-                                              GNUNET_CONNECTION_Credentials *
-                                              ucred,
-                                              const struct sockaddr * addr,
-                                              socklen_t addrlen);
-
-
-/**
- * Callback function for data received from the network.  Note that
- * both "available" and "err" would be 0 if the read simply timed out.
- *
- * @param cls closure
- * @param buf pointer to received data
- * @param available number of bytes availabe in "buf",
- *        possibly 0 (on errors)
- * @param addr address of the sender
- * @param addrlen size of addr
- * @param errCode value of errno (on errors receiving)
- */
-typedef void (*GNUNET_CONNECTION_Receiver) (void *cls, const void *buf,
-                                            size_t available,
-                                            const struct sockaddr * addr,
-                                            socklen_t addrlen, int errCode);
-
-/**
- * Set the persist option on this connection handle.  Indicates
- * that the underlying socket or fd should never really be closed.
- * Used for indicating process death.
- *
- * @param connection the connection to set persistent
- */
-void
-GNUNET_CONNECTION_persist_ (struct GNUNET_CONNECTION_Handle *connection);
-
-/**
- * Disable the "CORK" feature for communication with the given socket,
- * forcing the OS to immediately flush the buffer on transmission
- * instead of potentially buffering multiple messages.  Essentially
- * reduces the OS send buffers to zero.
- * Used to make sure that the last messages sent through the connection
- * reach the other side before the process is terminated.
- *
- * @param connection the connection to make flushing and blocking
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_CONNECTION_disable_corking (struct GNUNET_CONNECTION_Handle *connection);
-
-
-/**
- * Create a connection handle by (asynchronously) connecting to a host.
- * This function returns immediately, even if the connection has not
- * yet been established.  This function only creates TCP connections.
- *
- * @param s socket to connect
- * @param serv_addr server address
- * @param addrlen length of server address
- * @return the connection handle
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_connect_socket (struct GNUNET_NETWORK_Handle *s,
-                                  const struct sockaddr *serv_addr,
-                                  socklen_t addrlen);
-
-
-/**
- * Create a connection handle by boxing an existing OS socket.  The OS
- * socket should henceforth be no longer used directly.
- * #GNUNET_CONNECTION_destroy() will close it.
- *
- * @param osSocket existing socket to box
- * @return the boxed socket handle
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_from_existing (struct GNUNET_NETWORK_Handle *osSocket);
-
-
-/**
- * Create a connection handle by accepting on a listen socket.  This
- * function may block if the listen socket has no connection ready.
- *
- * @param access_cb function to use to check if access is allowed
- * @param access_cb_cls closure for @a access_cb
- * @param lsock listen socket
- * @return the connection handle, NULL on error (for example, access refused)
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_from_accept (GNUNET_CONNECTION_AccessCheck access_cb,
-                                      void *access_cb_cls,
-                                      struct GNUNET_NETWORK_Handle *lsock);
-
-
-/**
- * Create a connection handle by (asynchronously) connecting to a host.
- * This function returns immediately, even if the connection has not
- * yet been established.  This function only creates TCP connections.
- *
- * @param cfg configuration to use
- * @param hostname name of the host to connect to
- * @param port port to connect to
- * @return the connection handle
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_from_connect (const struct GNUNET_CONFIGURATION_Handle
-                                       *cfg, const char *hostname,
-                                       uint16_t port);
-
-
-/**
- * Create a connection handle by connecting to a UNIX domain service.
- * This function returns immediately, even if the connection has not
- * yet been established.  This function only creates UNIX connections.
- *
- * @param cfg configuration to use
- * @param unixpath path to connect to)
- * @return the connection handle, NULL on systems without UNIX support
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_from_connect_to_unixpath (const struct
-                                                   GNUNET_CONFIGURATION_Handle
-                                                   *cfg, const char *unixpath);
-
-
-
-
-/**
- * Create a connection handle by (asynchronously) connecting to a host.
- * This function returns immediately, even if the connection has not
- * yet been established.  This function only creates TCP connections.
- *
- * @param af_family address family to use
- * @param serv_addr server address
- * @param addrlen length of server address
- * @return the connection handle
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_from_sockaddr (int af_family,
-                                        const struct sockaddr *serv_addr,
-                                        socklen_t addrlen);
-
-/**
- * Check if connection is valid (no fatal errors have happened so far).
- * Note that a connection that is still trying to connect is considered
- * valid.
- *
- * @param connection handle to check
- * @return GNUNET_YES if valid, GNUNET_NO otherwise
- */
-int
-GNUNET_CONNECTION_check (struct GNUNET_CONNECTION_Handle *connection);
-
-
-/**
- * Obtain the network address of the other party.
- *
- * @param connection the client to get the address for
- * @param addr where to store the address
- * @param addrlen where to store the length of the address
- * @return GNUNET_OK on success
- */
-int
-GNUNET_CONNECTION_get_address (struct GNUNET_CONNECTION_Handle *connection,
-                               void **addr, size_t * addrlen);
-
-
-/**
- * Close the connection and free associated resources.  There must
- * not be any pending requests for reading or writing to the
- * connection at this time.
- *
- * @param connection connection to destroy
- */
-void
-GNUNET_CONNECTION_destroy (struct GNUNET_CONNECTION_Handle *connection);
-
-
-/**
- * Receive data from the given connection.  Note that this function will
- * call "receiver" asynchronously using the scheduler.  It will
- * "immediately" return.  Note that there MUST only be one active
- * receive call per connection at any given point in time (so do not
- * call receive again until the receiver callback has been invoked).
- *
- * @param connection connection handle
- * @param max maximum number of bytes to read
- * @param timeout maximum amount of time to wait
- * @param receiver function to call with received data
- * @param receiver_cls closure for receiver
- */
-void
-GNUNET_CONNECTION_receive (struct GNUNET_CONNECTION_Handle *connection, size_t max,
-                           struct GNUNET_TIME_Relative timeout,
-                           GNUNET_CONNECTION_Receiver receiver,
-                           void *receiver_cls);
-
-
-/**
- * Cancel receive job on the given connection.  Note that the
- * receiver callback must not have been called yet in order
- * for the cancellation to be valid.
- *
- * @param connection connection handle
- * @return closure of the original receiver callback closure
- */
-void *
-GNUNET_CONNECTION_receive_cancel (struct GNUNET_CONNECTION_Handle *connection);
-
-
-/**
- * Function called to notify a client about the connection begin ready
- * to queue more data.  @a buf will be NULL and @a size zero if the
- * connection was closed for writing in the meantime.
- *
- * @param cls closure
- * @param size number of bytes available in @a buf
- * @param buf where the callee should write the message
- * @return number of bytes written to @a buf
- */
-typedef size_t
-(*GNUNET_CONNECTION_TransmitReadyNotify) (void *cls,
-                                          size_t size,
-                                          void *buf);
-
-
-/**
- * Opaque handle that can be used to cancel
- * a transmit-ready notification.
- */
-struct GNUNET_CONNECTION_TransmitHandle;
-
-/**
- * Ask the connection to call us once the specified number of bytes
- * are free in the transmission buffer.  Will never call the @a notify
- * callback in this task, but always first go into the scheduler.  Note that
- * this function will abort if "size" is greater than
- * #GNUNET_SERVER_MAX_MESSAGE_SIZE.
- *
- * Note that "notify" will be called either when enough
- * buffer space is available OR when the connection is destroyed.
- * The size parameter given to notify is guaranteed to be
- * larger or equal to size if the buffer is ready, or zero
- * if the connection was destroyed (or at least closed for
- * writing).  Finally, any time before 'notify' is called, a
- * client may call "notify_transmit_ready_cancel" to cancel
- * the transmission request.
- *
- * Only one transmission request can be scheduled at the same
- * time.  Notify will be run with the same scheduler priority
- * as that of the caller.
- *
- * @param connection connection
- * @param size number of bytes to send
- * @param timeout after how long should we give up (and call
- *        notify with buf NULL and size 0)?
- * @param notify function to call when buffer space is available
- * @param notify_cls closure for notify
- * @return non-NULL if the notify callback was queued,
- *         NULL if we are already going to notify someone else (busy)
- */
-struct GNUNET_CONNECTION_TransmitHandle *
-GNUNET_CONNECTION_notify_transmit_ready (struct GNUNET_CONNECTION_Handle *connection,
-                                         size_t size,
-                                         struct GNUNET_TIME_Relative timeout,
-                                         GNUNET_CONNECTION_TransmitReadyNotify
-                                         notify, void *notify_cls);
-
-
-/**
- * Cancel the specified transmission-ready
- * notification.
- *
- * @param th handle for notification to cancel
- */
-void
-GNUNET_CONNECTION_notify_transmit_ready_cancel (struct
-                                                GNUNET_CONNECTION_TransmitHandle
-                                                *th);
-
-
-/**
- * Create a connection to be proxied using a given connection.
- *
- * @param cph connection to proxy server
- * @return connection to be proxied
- */
-struct GNUNET_CONNECTION_Handle *
-GNUNET_CONNECTION_create_proxied_from_handshake (struct GNUNET_CONNECTION_Handle *cph);
-
-
-/**
- * Activate proxied connection and destroy initial proxy handshake connection.
- * There must not be any pending requests for reading or writing to the
- * proxy hadshake connection at this time.
- *
- * @param proxied connection connection to proxy server
- */
-void
-GNUNET_CONNECTION_acivate_proxied (struct GNUNET_CONNECTION_Handle *proxied);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_CONNECTION_LIB_H */
-#endif
-
-/** @} */  /* end of group */
-
-/* end of gnunet_connection_lib.h */
diff --git a/src/include/gnunet_constants.h b/src/include/gnunet_constants.h
index 1d0232cea..1c48298a0 100644
--- a/src/include/gnunet_constants.h
+++ b/src/include/gnunet_constants.h
@@ -40,6 +40,8 @@ extern "C"
 #endif
 #endif
 
+
+
 /**
  * Bandwidth (in/out) to assume initially (before either peer has
  * communicated any particular preference).  Should be rather low; set
diff --git a/src/include/gnunet_core_service.h b/src/include/gnunet_core_service.h
index 8136770b7..ace223c11 100644
--- a/src/include/gnunet_core_service.h
+++ b/src/include/gnunet_core_service.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009-2014, 2016 GNUnet e.V.
+     Copyright (C) 2009-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -197,57 +197,6 @@ GNUNET_CORE_get_mq (const struct GNUNET_CORE_Handle *h,
 
 
 /**
- * Handle for a transmission request.
- */
-struct GNUNET_CORE_TransmitHandle;
-
-
-/**
- * Ask the core to call @a notify once it is ready to transmit the
- * given number of bytes to the specified @a target.  Must only be
- * called after a connection to the respective peer has been
- * established (and the client has been informed about this).  You may
- * have one request of this type pending for each connected peer at
- * any time.  If a peer disconnects, the application MUST call
- * #GNUNET_CORE_notify_transmit_ready_cancel() on the respective
- * transmission request, if one such request is pending.
- *
- * @param handle connection to core service
- * @param cork is corking allowed for this transmission?
- * @param priority how important is the message?
- * @param maxdelay how long can the message wait? Only effective if @a cork is #GNUNET_YES
- * @param target who should receive the message, never NULL (can be this peer's identity for loopback)
- * @param notify_size how many bytes of buffer space does @a notify want?
- * @param notify function to call when buffer space is available;
- *        will be called with NULL on timeout; clients MUST cancel
- *        all pending transmission requests DURING the disconnect
- *        handler
- * @param notify_cls closure for @a notify
- * @return non-NULL if the notify callback was queued,
- *         NULL if we can not even queue the request (request already pending);
- *         if NULL is returned, @a notify will NOT be called.
- */
-struct GNUNET_CORE_TransmitHandle *
-GNUNET_CORE_notify_transmit_ready (struct GNUNET_CORE_Handle *handle,
-                                   int cork,
-                                   enum GNUNET_CORE_Priority priority,
-                                   struct GNUNET_TIME_Relative maxdelay,
-                                   const struct GNUNET_PeerIdentity *target,
-                                   size_t notify_size,
-                                   GNUNET_CONNECTION_TransmitReadyNotify notify,
-                                   void *notify_cls);
-
-
-/**
- * Cancel the specified transmission-ready notification.
- *
- * @param th handle that was returned by #GNUNET_CORE_notify_transmit_ready().
- */
-void
-GNUNET_CORE_notify_transmit_ready_cancel (struct GNUNET_CORE_TransmitHandle *th);
-
-
-/**
  * Handle to a CORE monitoring operation.
  */
 struct GNUNET_CORE_MonitorHandle;
diff --git a/src/include/gnunet_datastore_plugin.h b/src/include/gnunet_datastore_plugin.h
index 2295d4e72..b1c9cb7c3 100644
--- a/src/include/gnunet_datastore_plugin.h
+++ b/src/include/gnunet_datastore_plugin.h
@@ -204,9 +204,9 @@ typedef void
  * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
- * @param key key to match, never NULL
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
+ * @param key maybe NULL (to match all entries)
  * @param vhash hash of the value, maybe NULL (to
  *        match all values that have the right key).
  *        Note that for DBlocks there is no difference
@@ -215,17 +215,18 @@ typedef void
  * @param type entries of which type are relevant?
  *     Use 0 for any type.
  * @param proc function to call on the matching value;
- *        proc should be called with NULL if there is no result
+ *        will be called with NULL if nothing matches
  * @param proc_cls closure for @a proc
  */
 typedef void
 (*PluginGetKey) (void *cls,
-                 uint64_t offset,
-                 const struct GNUNET_HashCode *key,
-                 const struct GNUNET_HashCode *vhash,
-                 enum GNUNET_BLOCK_Type type,
-                 PluginDatumProcessor proc,
-                 void *proc_cls);
+                 uint64_t next_uid,
+                 bool random,
+                 const struct GNUNET_HashCode *key,
+                 const struct GNUNET_HashCode *vhash,
+                 enum GNUNET_BLOCK_Type type,
+                 PluginDatumProcessor proc,
+                 void *proc_cls);
 
 
 /**
@@ -285,23 +286,22 @@ typedef void
 
 
 /**
- * Select a single item from the datastore at the specified offset
- * (among those applicable).
+ * Select a single item from the datastore (among those applicable).
  *
  * @param cls closure
- * @param offset offset of the result (modulo num-results);
- *               specific ordering does not matter for the offset
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param type entries of which type should be considered?
  *        Must not be zero (ANY).
- * @param proc function to call on the matching value
+ * @param proc function to call on the matching value;
+ *        will be called with NULL if no value matches
  * @param proc_cls closure for @a proc
  */
 typedef void
 (*PluginGetType) (void *cls,
-                  uint64_t offset,
-                  enum GNUNET_BLOCK_Type type,
-                  PluginDatumProcessor proc,
-                  void *proc_cls);
+                  uint64_t next_uid,
+                  enum GNUNET_BLOCK_Type type,
+                  PluginDatumProcessor proc,
+                  void *proc_cls);
 
 
 /**
@@ -354,9 +354,6 @@ struct GNUNET_DATASTORE_PluginFunctions
 
   /**
    * Get datum (of the specified type) with anonymity level zero.
-   * This function is allowed to ignore the 'offset' argument
-   * and instead return a random result (with zero anonymity of
-   * the correct type) if implementing an offset is expensive.
    */
   PluginGetType get_zero_anonymity;
 
diff --git a/src/include/gnunet_datastore_service.h b/src/include/gnunet_datastore_service.h
index f594d8fa6..830e7da86 100644
--- a/src/include/gnunet_datastore_service.h
+++ b/src/include/gnunet_datastore_service.h
@@ -201,33 +201,6 @@ GNUNET_DATASTORE_release_reserve (struct GNUNET_DATASTORE_Handle *h,
 
 
 /**
- * Update a value in the datastore.
- *
- * @param h handle to the datastore
- * @param uid identifier for the value
- * @param priority how much to increase the priority of the value
- * @param expiration new expiration value should be MAX of existing and this argument
- * @param queue_priority ranking of this request in the priority queue
- * @param max_queue_size at what queue size should this request be dropped
- *        (if other requests of higher priority are in the queue)
- * @param cont continuation to call when done
- * @param cont_cls closure for @a cont
- * @return NULL if the entry was not queued, otherwise a handle that can be used to
- *         cancel; note that even if NULL is returned, the callback will be invoked
- *         (or rather, will already have been invoked)
- */
-struct GNUNET_DATASTORE_QueueEntry *
-GNUNET_DATASTORE_update (struct GNUNET_DATASTORE_Handle *h,
-                         uint64_t uid,
-                         uint32_t priority,
-                         struct GNUNET_TIME_Absolute expiration,
-                         unsigned int queue_priority,
-                         unsigned int max_queue_size,
-                         GNUNET_DATASTORE_ContinuationWithStatus cont,
-                         void *cont_cls);
-
-
-/**
  * Explicitly remove some content from the database.  @a cont will be
  * called with status #GNUNET_OK if content was removed, #GNUNET_NO if
  * no matching entry was found and #GNUNET_SYSERR on all other types
@@ -288,10 +261,8 @@ typedef void
  * will only be called once.
  *
  * @param h handle to the datastore
- * @param offset offset of the result (modulo num-results); set to
- *               a random 64-bit value initially; then increment by
- *               one each time; detect that all results have been found by uid
- *               being again the first uid ever returned.
+ * @param next_uid return the result with lowest uid >= next_uid
+ * @param random if true, return a random result instead of using next_uid
  * @param key maybe NULL (to match all entries)
  * @param type desired type, 0 for any
  * @param queue_priority ranking of this request in the priority queue
@@ -305,7 +276,8 @@ typedef void
  */
 struct GNUNET_DATASTORE_QueueEntry *
 GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
-                          uint64_t offset,
+                          uint64_t next_uid,
+                          bool random,
                           const struct GNUNET_HashCode *key,
                           enum GNUNET_BLOCK_Type type,
                           unsigned int queue_priority,
@@ -316,16 +288,9 @@ GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
 
 /**
  * Get a single zero-anonymity value from the datastore.
- * Note that some implementations can ignore the 'offset' and
- * instead return a random zero-anonymity value.  In that case,
- * detecting the wrap-around based on a repeating UID is at best
- * probabilistic.
  *
  * @param h handle to the datastore
- * @param offset offset of the result (modulo num-results); set to
- *               a random 64-bit value initially; then increment by
- *               one each time; detect that all results have been found by uid
- *               being again the first uid ever returned.
+ * @param next_uid return the result with lowest uid >= next_uid
  * @param queue_priority ranking of this request in the priority queue
  * @param max_queue_size at what queue size should this request be dropped
  *        (if other requests of higher priority are in the queue)
@@ -339,7 +304,7 @@ GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h,
  */
 struct GNUNET_DATASTORE_QueueEntry *
 GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h,
-                                     uint64_t offset,
+                                     uint64_t next_uid,
                                      unsigned int queue_priority,
                                      unsigned int max_queue_size,
                                      enum GNUNET_BLOCK_Type type,
diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h
index a9c7e8944..ac418072e 100644
--- a/src/include/gnunet_fs_service.h
+++ b/src/include/gnunet_fs_service.h
@@ -438,23 +438,36 @@ GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_CONTAINER_MetaData
 /* ******************** command-line option parsing API *********************** */
 
 /**
- * Command-line option parser function that allows the user
- * to specify one or more '-k' options with keywords.  Each
- * specified keyword will be added to the URI.  A pointer to
- * the URI must be passed as the "scls" argument.
- *
- * @param ctx command line processor context
- * @param scls must be of type "struct GNUNET_FS_Uri **"
- * @param option name of the option (typically 'k')
- * @param value command line argument given
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                               void *scls,
-                               const char *option,
-                               const char *value);
-
+ * Allow user to specify keywords.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] topKeywords set to the desired value
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_FS_GETOPT_KEYWORDS (char shortName,
+                           const char *name,
+                           const char *argumentHelp,
+                           const char *description,
+                           struct GNUNET_FS_Uri **topKeywords);
+
+/**
+ * Allow user to specify metadata.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] metadata set to the desired value
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_FS_GETOPT_METADATA (char shortName,
+                           const char *name,
+                           const char *argumentHelp,
+                           const char *description,
+                           struct GNUNET_CONTAINER_MetaData **meta);
 
 /**
  * Command-line option parser function that allows the user to specify
diff --git a/src/include/gnunet_getopt_lib.h b/src/include/gnunet_getopt_lib.h
index b04020a70..c2bd72340 100644
--- a/src/include/gnunet_getopt_lib.h
+++ b/src/include/gnunet_getopt_lib.h
@@ -97,6 +97,7 @@ typedef int
                                              const char *option,
                                              const char *value);
 
+
 /**
  * @brief Definition of a command line option.
  */
@@ -130,255 +131,301 @@ struct GNUNET_GETOPT_CommandLineOption
   int require_argument;
 
   /**
+   * Is the presence of this option mandatory?
+   */
+  int option_mandatory;
+
+  /**
    * Handler for the option.
    */
   GNUNET_GETOPT_CommandLineOptionProcessor processor;
 
   /**
+   * Function to call on @e scls to clean up after processing all
+   * the arguments. Can be NULL.
+   */
+  void (*cleaner)(void *cls);
+
+  /**
    * Specific closure to pass to the processor.
    */
   void *scls;
 
 };
 
+
 /**
- * Macro defining the option to print the command line
+ * Defining the option to print the command line
  * help text (-h option).
  *
  * @param about string with brief description of the application
  */
-#define GNUNET_GETOPT_OPTION_HELP(about) \
-  { 'h', "help", (const char *) NULL, gettext_noop("print this help"), 0, &GNUNET_GETOPT_format_help_, (void *) about }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_HELP (const char *about);
 
 
 /**
- * Macro defining the option to print the version of
+ * Define the option to print the version of
  * the application (-v option)
  *
  * @param version string with the version number
  */
-#define GNUNET_GETOPT_OPTION_VERSION(version) \
-  { 'v', "version", (const char *) NULL, gettext_noop("print the version number"), 0, &GNUNET_GETOPT_print_version_, (void *) version }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_VERSION (const char *version);
+
 
 
 /**
  * Allow user to specify log file name (-l option)
  *
- * @param logfn set to the name of the logfile
+ * @param[out] logfn set to the name of the logfile
  */
-#define GNUNET_GETOPT_OPTION_LOGFILE(logfn)                             \
-  { 'l', "logfile", "LOGFILE", gettext_noop("configure logging to write logs to LOGFILE"), 1, &GNUNET_GETOPT_set_string, (void *) logfn }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_LOGFILE (char **logfn);
 
 
 /**
- * Allow user to specify log level (-L option)
+ * Allow user to specify a string.
  *
- * @param loglev set to the log level
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] str set to the string
  */
-#define GNUNET_GETOPT_OPTION_LOGLEVEL(loglev)                           \
-  { 'L', "log", "LOGLEVEL", gettext_noop("configure logging to use LOGLEVEL"), 1, &GNUNET_GETOPT_set_string, (void *) loglev }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_STRING (char shortName,
+                             const char *name,
+                             const char *argumentHelp,
+                             const char *description,
+                             char **str);
+
+/**
+ * Allow user to specify a filename (automatically path expanded).
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] str set to the string
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_FILENAME (char shortName,
+                               const char *name,
+                               const char *argumentHelp,
+                               const char *description,
+                               char **str);
 
 
 /**
- * Get number of verbose (-V) flags
+ * Allow user to specify a binary value using Crockford
+ * Base32 encoding.
  *
- * @param level where to store the verbosity level (should be an 'int')
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val binary value decoded from Crockford Base32-encoded argument
+ * @param val_size size of @a val in bytes
  */
-#define GNUNET_GETOPT_OPTION_VERBOSE(level)                             \
-  { 'V', "verbose", (const char *) NULL, gettext_noop("be verbose"), 0, &GNUNET_GETOPT_increment_value, (void *) level }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_BASE32_FIXED_SIZE (char shortName,
+                                            const char *name,
+                                            const char *argumentHelp,
+                                            const char *description,
+                                            void *val,
+                                            size_t val_size);
 
 
 /**
- * Get configuration file name (-c option)
+ * Allow user to specify a binary value using Crockford
+ * Base32 encoding where the size of the binary value is
+ * automatically determined from its type.
  *
- * @param fn set to the configuration file name
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val binary value decoded from Crockford Base32-encoded argument;
+ *             size is determined by type (sizeof (*val)).
  */
-#define GNUNET_GETOPT_OPTION_CFG_FILE(fn)                               \
-  { 'c', "config", "FILENAME", gettext_noop("use configuration file FILENAME"), 1, &GNUNET_GETOPT_set_string, (void *) fn }
+#define GNUNET_GETOPT_OPTION_SET_BASE32_AUTO(shortName,name,argumentHelp,description,val) \
+  GNUNET_GETOPT_OPTION_SET_BASE32_FIXED_SIZE(shortName,name,argumentHelp,description,val,sizeof(*val))
 
 
 /**
- * Marker for the end of the list of options.
+ * Allow user to specify a flag (which internally means setting
+ * an integer to 1/#GNUNET_YES/#GNUNET_OK.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param description long help text for the option
+ * @param[out] val set to 1 if the option is present
  */
-#define GNUNET_GETOPT_OPTION_END \
-  { '\0', NULL, NULL, NULL, 0, NULL, NULL }
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_ONE (char shortName,
+                              const char *name,
+                              const char *description,
+                              int *val);
 
 
 /**
- * Parse the command line.
+ * Allow user to specify an `unsigned int`.
  *
- * @param binaryOptions Name of application with option summary
- * @param allOptions defined options and handlers
- * @param argc number of arguments in @a argv
- * @param argv actual arguments
- * @return index into argv with first non-option
- *   argument, or #GNUNET_SYSERR on error
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the value specified at the command line
  */
-int
-GNUNET_GETOPT_run (const char *binaryOptions,
-                   const struct GNUNET_GETOPT_CommandLineOption *allOptions,
-                   unsigned int argc, char *const *argv);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_UINT (char shortName,
+                               const char *name,
+                               const char *argumentHelp,
+                               const char *description,
+                               unsigned int *val);
 
 
 /**
- * Set an option of type 'unsigned long long' from the command line.
- * A pointer to this function should be passed as part of the
- * `struct GNUNET_GETOPT_CommandLineOption` array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type `unsigned long long`.
+ * Allow user to specify an `unsigned long long`.
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'unsigned long long')
- * @param option name of the option
- * @param value actual value of the option as a string.
- * @return #GNUNET_OK if parsing the value worked
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the value specified at the command line
  */
-int
-GNUNET_GETOPT_set_ulong (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                         void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_ULONG (char shortName,
+                                const char *name,
+                                const char *argumentHelp,
+                                const char *description,
+                                unsigned long long *val);
 
 
 /**
- * Set an option of type 'struct GNUNET_TIME_Relative' from the command line.
- * A pointer to this function should be passed as part of the
- * `struct GNUNET_GETOPT_CommandLineOption` array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type `struct GNUNET_TIME_Relative`.
+ * Allow user to specify a `struct GNUNET_TIME_Relative`
+ * (using human-readable "fancy" time).
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'struct GNUNET_TIME_Relative')
- * @param option name of the option
- * @param value actual value of the option as a string.
- * @return #GNUNET_OK if parsing the value worked
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the time specified at the command line
  */
-int
-GNUNET_GETOPT_set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                                 void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME (char shortName,
+                                        const char *name,
+                                        const char *argumentHelp,
+                                        const char *description,
+                                        struct GNUNET_TIME_Relative *val);
 
 
 /**
- * Set an option of type 'unsigned int' from the command line.
- * A pointer to this function should be passed as part of the
- * `struct GNUNET_GETOPT_CommandLineOption` array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type `unsigned int`.
+ * Allow user to specify a `struct GNUNET_TIME_Absolute`
+ * (using human-readable "fancy" time).
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'unsigned int')
- * @param option name of the option
- * @param value actual value of the option as a string.
- * @return #GNUNET_OK if parsing the value worked
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the time specified at the command line
  */
-int
-GNUNET_GETOPT_set_uint (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                        void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_ABSOLUTE_TIME (char shortName,
+                                        const char *name,
+                                        const char *argumentHelp,
+                                        const char *description,
+                                        struct GNUNET_TIME_Absolute *val);
 
 
 /**
- * Set an option of type 'int' from the command line to 1 if the
- * given option is present.
- * A pointer to this function should be passed as part of the
- * `struct GNUNET_GETOPT_CommandLineOption` array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type `int`.
+ * Increment @a val each time the option flag is given by one.
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the `int`)
- * @param option name of the option
- * @param value not used (NULL)
- * @return #GNUNET_OK (always)
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to 1 if the option is present
  */
-int
-GNUNET_GETOPT_set_one (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                       void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_INCREMENT_VALUE (char shortName,
+                                      const char *name,
+                                      const char *description,
+                                      unsigned int *val);
 
 
 /**
- * Set an option of type 'char *' from the command line.
- * A pointer to this function should be passed as part of the
- * `struct GNUNET_GETOPT_CommandLineOption` array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type `char *`, which will be allocated with the requested string.
+ * Define the '-L' log level option.  Note that we do not check
+ * that the log level is valid here.
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the `char *`,
- *             which will be allocated)
- * @param option name of the option
- * @param value actual value of the option (a string)
- * @return #GNUNET_OK (always)
+ * @param[out] level set to the log level
  */
-int
-GNUNET_GETOPT_set_string (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                          void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_LOGLEVEL (char **level);
 
 
 /**
- * Set an option of type 'char *' from the command line doing fs expansion.
- * A pointer to this function should be passed as part of the
- * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type 'char *', which will be allocated with the requested string.
+ * Define the '-V' verbosity option.  Using the option more
+ * than once increments @a level each time.
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'char *',
- *             which will be allocated)
- * @param option name of the option
- * @param value actual value of the option (a string)
- * @return #GNUNET_OK (always)
+ * @param[out] level set to the verbosity level
  */
-int
-GNUNET_GETOPT_set_filename (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                            void *scls, const char *option, const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_VERBOSE (unsigned int *level);
+
 
 /**
- * Set an option of type 'unsigned int' from the command line. Each
- * time the option flag is given, the value is incremented by one.
- * A pointer to this function should be passed as part of the
- * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type 'int'.
+ * Allow user to specify log file name (-l option)
  *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'int')
- * @param option name of the option
- * @param value not used (NULL)
- * @return #GNUNET_OK (always)
+ * @param[out] logfn set to the name of the logfile
  */
-int
-GNUNET_GETOPT_increment_value (struct GNUNET_GETOPT_CommandLineProcessorContext
-                               *ctx, void *scls, const char *option,
-                               const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_LOGFILE (char **logfn);
 
 
-/* *************** internal prototypes - use macros above! ************* */
+/**
+ * Allow user to specify configuration file name (-c option)
+ *
+ * @param[out] fn set to the name of the configuration file
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_CFG_FILE (char **fn);
+
 
 /**
- * Print out details on command line options (implements --help).
+ * Make the given option mandatory.
  *
- * @param ctx command line processing context
- * @param scls additional closure (points to about text)
- * @param option name of the option
- * @param value not used (NULL)
- * @return #GNUNET_NO (do not continue, not an error)
+ * @param opt option to modify
+ * @return @a opt with the mandatory flag set.
  */
-int
-GNUNET_GETOPT_format_help_ (struct GNUNET_GETOPT_CommandLineProcessorContext
-                            *ctx, void *scls, const char *option,
-                            const char *value);
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_MANDATORY (struct GNUNET_GETOPT_CommandLineOption opt);
+
 
 /**
- * Print out program version (implements --version).
+ * Marker for the end of the list of options.
+ */
+#define GNUNET_GETOPT_OPTION_END \
+  { '\0', NULL, NULL, NULL, 0, 0, NULL, NULL, NULL }
+
+
+/**
+ * Parse the command line.
  *
- * @param ctx command line processing context
- * @param scls additional closure (points to version string)
- * @param option name of the option
- * @param value not used (NULL)
- * @return #GNUNET_NO (do not continue, not an error)
+ * @param binaryOptions Name of application with option summary
+ * @param allOptions defined options and handlers
+ * @param argc number of arguments in @a argv
+ * @param argv actual arguments
+ * @return index into argv with first non-option
+ *   argument, or #GNUNET_SYSERR on error
  */
 int
-GNUNET_GETOPT_print_version_ (struct GNUNET_GETOPT_CommandLineProcessorContext
-                              *ctx, void *scls, const char *option,
-                              const char *value);
+GNUNET_GETOPT_run (const char *binaryOptions,
+                   const struct GNUNET_GETOPT_CommandLineOption *allOptions,
+                   unsigned int argc,
+                   char *const *argv);
+
 
 #if 0                           /* keep Emacsens' auto-indent happy */
 {
diff --git a/src/include/gnunet_helper_lib.h b/src/include/gnunet_helper_lib.h
index db0ca98aa..60b3ff681 100644
--- a/src/include/gnunet_helper_lib.h
+++ b/src/include/gnunet_helper_lib.h
@@ -38,7 +38,8 @@
 #define GNUNET_HELPER_LIB_H
 
 #include "gnunet_scheduler_lib.h"
-#include "gnunet_server_lib.h"
+#include "gnunet_mst_lib.h"
+
 
 /**
  * The handle to a helper process.
@@ -52,7 +53,8 @@ struct GNUNET_HELPER_Handle;
  *
  * @param cls the closure from GNUNET_HELPER_start()
  */
-typedef void (*GNUNET_HELPER_ExceptionCallback) (void *cls);
+typedef void
+(*GNUNET_HELPER_ExceptionCallback) (void *cls);
 
 
 /**
@@ -75,7 +77,7 @@ struct GNUNET_HELPER_Handle *
 GNUNET_HELPER_start (int with_control_pipe,
                      const char *binary_name,
                      char *const binary_argv[],
-                     GNUNET_SERVER_MessageTokenizerCallback cb,
+                     GNUNET_MessageTokenizerCallback cb,
                      GNUNET_HELPER_ExceptionCallback exp_cb,
                      void *cb_cls);
 
diff --git a/src/include/gnunet_json_lib.h b/src/include/gnunet_json_lib.h
index 9e5f9e284..ce721d8d8 100644
--- a/src/include/gnunet_json_lib.h
+++ b/src/include/gnunet_json_lib.h
@@ -427,6 +427,26 @@ void
 GNUNET_JSON_post_parser_cleanup (void *con_cls);
 
 
+/* ****************** GETOPT JSON helper ******************* */
+
+
+/**
+ * Allow user to specify a JSON input value.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the JSON specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_JSON_getopt (char shortName,
+                    const char *name,
+                    const char *argumentHelp,
+                    const char *description,
+                    json_t **json);
+
+
 #endif
 
 /* end of gnunet_json_lib.h */
diff --git a/src/include/gnunet_mq_lib.h b/src/include/gnunet_mq_lib.h
index b05128ccc..ecee1b223 100644
--- a/src/include/gnunet_mq_lib.h
+++ b/src/include/gnunet_mq_lib.h
@@ -35,6 +35,7 @@
 #ifndef GNUNET_MQ_LIB_H
 #define GNUNET_MQ_LIB_H
 
+#include "gnunet_scheduler_lib.h"
 
 /**
  * Allocate an envelope, with extra space allocated after the space needed
diff --git a/src/include/gnunet_multicast_service.h b/src/include/gnunet_multicast_service.h
index 3829a7040..5645207dd 100644
--- a/src/include/gnunet_multicast_service.h
+++ b/src/include/gnunet_multicast_service.h
@@ -617,6 +617,7 @@ GNUNET_MULTICAST_origin_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
  *         data.  If 0 is returned in @a data_size the transmission is paused,
  *         and can be resumed with GNUNET_MULTICAST_origin_to_all_resume().
  *         #GNUNET_YES if this completes the transmission (all data supplied)
+ * @deprecated should move to MQ-style API!
  */
 typedef int
 (*GNUNET_MULTICAST_OriginTransmitNotify) (void *cls,
@@ -647,6 +648,7 @@ struct GNUNET_MULTICAST_OriginTransmitHandle;
  *        Closure for @a notify.
  *
  * @return NULL on error (i.e. request already pending).
+ * @deprecated should move to MQ-style API!
  */
 struct GNUNET_MULTICAST_OriginTransmitHandle *
 GNUNET_MULTICAST_origin_to_all (struct GNUNET_MULTICAST_Origin *origin,
@@ -854,6 +856,7 @@ GNUNET_MULTICAST_member_part (struct GNUNET_MULTICAST_Member *member,
  *         data.  If 0 is returned in @a data_size the transmission is paused,
  *         and can be resumed with GNUNET_MULTICAST_member_to_origin_resume().
  *         #GNUNET_YES if this completes the transmission (all data supplied)
+ * @deprecated should move to MQ-style API!
  */
 typedef int
 (*GNUNET_MULTICAST_MemberTransmitNotify) (void *cls,
@@ -880,6 +883,7 @@ struct GNUNET_MULTICAST_MemberTransmitHandle;
  *        Closure for @a notify.
  *
  * @return Handle to cancel request, NULL on error (i.e. request already pending).
+ * @deprecated should move to MQ-style API!
  */
 struct GNUNET_MULTICAST_MemberTransmitHandle *
 GNUNET_MULTICAST_member_to_origin (struct GNUNET_MULTICAST_Member *member,
diff --git a/src/include/gnunet_peerstore_service.h b/src/include/gnunet_peerstore_service.h
index 3cafe70b8..c636447c2 100644
--- a/src/include/gnunet_peerstore_service.h
+++ b/src/include/gnunet_peerstore_service.h
@@ -86,7 +86,7 @@ struct GNUNET_PEERSTORE_Record
   /**
    * Peer Identity
    */
-  struct GNUNET_PeerIdentity *peer;
+  struct GNUNET_PeerIdentity peer;
 
   /**
    * Record key string
@@ -106,7 +106,7 @@ struct GNUNET_PEERSTORE_Record
   /**
    * Expiry time of entry
    */
-  struct GNUNET_TIME_Absolute *expiry;
+  struct GNUNET_TIME_Absolute expiry;
 
   /**
    * Client from which this record originated.
diff --git a/src/include/gnunet_server_lib.h b/src/include/gnunet_server_lib.h
deleted file mode 100644
index 5da31bcd7..000000000
--- a/src/include/gnunet_server_lib.h
+++ /dev/null
@@ -1,887 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009-2013 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @author Christian Grothoff
- *
- * @file
- * Library for building GNUnet network servers
-
- * @defgroup server  Server library
- * Library for building GNUnet network servers
- *
- * Provides functions for a server that communicates with clients.
- *
- * @see [Documentation](https://gnunet.org/ipc)
- *
- * @{
- */
-
-#ifndef GNUNET_SERVER_LIB_H
-#define GNUNET_SERVER_LIB_H
-
-#ifdef __cplusplus
-extern "C"
-{
-#if 0                           /* keep Emacsens' auto-indent happy */
-}
-#endif
-#endif
-
-#include "gnunet_common.h"
-#include "gnunet_connection_lib.h"
-
-
-/**
- * Largest supported message (to be precise, one byte more
- * than the largest possible message, so tests involving
- * this value should check for messages being smaller than
- * this value).
- */
-#define GNUNET_SERVER_MAX_MESSAGE_SIZE 65536
-
-/**
- * Smallest supported message.
- */
-#define GNUNET_SERVER_MIN_BUFFER_SIZE sizeof (struct GNUNET_MessageHeader)
-
-/**
- * @brief handle for a server
- */
-struct GNUNET_SERVER_Handle;
-
-/**
- * @brief opaque handle for a client of the server
- */
-struct GNUNET_SERVER_Client;
-
-/**
- * @brief opaque handle server returns for aborting transmission to a client.
- */
-struct GNUNET_SERVER_TransmitHandle;
-
-
-/**
- * Functions with this signature are called whenever a message is
- * received.
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- */
-typedef void
-(*GNUNET_SERVER_MessageCallback) (void *cls,
-                                  struct GNUNET_SERVER_Client *client,
-                                  const struct GNUNET_MessageHeader *message);
-
-
-/**
- * Message handler.  Each struct specifies how to handle on particular
- * type of message received.
- */
-struct GNUNET_SERVER_MessageHandler
-{
-  /**
-   * Function to call for messages of "type".
-   */
-  GNUNET_SERVER_MessageCallback callback;
-
-  /**
-   * Closure argument for @e callback.
-   */
-  void *callback_cls;
-
-  /**
-   * Type of the message this handler covers.
-   */
-  uint16_t type;
-
-  /**
-   * Expected size of messages of this type.  Use 0 for
-   * variable-size.  If non-zero, messages of the given
-   * type will be discarded (and the connection closed)
-   * if they do not have the right size.
-   */
-  uint16_t expected_size;
-
-};
-
-
-/**
- * Create a new server.
- *
- * @param access_cb function for access control
- * @param access_cb_cls closure for @a access_cb
- * @param lsocks NULL-terminated array of listen sockets
- * @param idle_timeout after how long should we timeout idle connections?
- * @param require_found if #GNUNET_YES, connections sending messages of unknown type
- *        will be closed
- * @return handle for the new server, NULL on error
- *         (typically, "port" already in use)
- */
-struct GNUNET_SERVER_Handle *
-GNUNET_SERVER_create_with_sockets (GNUNET_CONNECTION_AccessCheck access_cb,
-                                   void *access_cb_cls,
-                                   struct GNUNET_NETWORK_Handle **lsocks,
-                                   struct GNUNET_TIME_Relative idle_timeout,
-                                   int require_found);
-
-/**
- * Create a new server.
- *
- * @param access_cb function for access control
- * @param access_cb_cls closure for @a access_cb
- * @param server_addr address toes listen on (including port), NULL terminated array
- * @param socklen lengths of respective @a server_addr
- * @param idle_timeout after how long should we timeout idle connections?
- * @param require_found if #GNUNET_YES, connections sending messages of unknown type
- *        will be closed
- * @return handle for the new server, NULL on error
- *         (typically, "port" already in use)
- */
-struct GNUNET_SERVER_Handle *
-GNUNET_SERVER_create (GNUNET_CONNECTION_AccessCheck access_cb,
-                      void *access_cb_cls,
-                      struct sockaddr *const *server_addr,
-                      const socklen_t *socklen,
-                      struct GNUNET_TIME_Relative idle_timeout,
-                      int require_found);
-
-
-/**
- * Suspend accepting connections from the listen socket temporarily.
- * Resume activity using #GNUNET_SERVER_resume.
- *
- * @param server server to stop accepting connections.
- */
-void
-GNUNET_SERVER_suspend (struct GNUNET_SERVER_Handle *server);
-
-
-/**
- * Resume accepting connections from the listen socket.
- *
- * @param server server to resume accepting connections.
- */
-void
-GNUNET_SERVER_resume (struct GNUNET_SERVER_Handle *server);
-
-
-/**
- * Stop the listen socket and get ready to shutdown the server once
- * only clients marked using #GNUNET_SERVER_client_mark_monitor are
- * left.
- *
- * @param server server to stop listening on
- */
-void
-GNUNET_SERVER_stop_listening (struct GNUNET_SERVER_Handle *server);
-
-
-/**
- * Free resources held by this server.
- *
- * @param server server to destroy
- */
-void
-GNUNET_SERVER_destroy (struct GNUNET_SERVER_Handle *server);
-
-
-/**
- * Add additional handlers to an existing server.
- *
- * @param server the server to add handlers to
- * @param handlers array of message handlers for
- *        incoming messages; the last entry must
- *        have "NULL" for the "callback"; multiple
- *        entries for the same type are allowed,
- *        they will be called in order of occurence.
- *        These handlers can be removed later;
- *        the handlers array must exist until removed
- *        (or server is destroyed).
- */
-void
-GNUNET_SERVER_add_handlers (struct GNUNET_SERVER_Handle *server,
-                            const struct GNUNET_SERVER_MessageHandler *handlers);
-
-
-/**
- * Notify us when the server has enough space to transmit
- * a message of the given size to the given client.
- *
- * @param client client to transmit message to
- * @param size requested amount of buffer space
- * @param timeout after how long should we give up (and call
- *        notify with buf NULL and size 0)?
- * @param callback function to call when space is available
- * @param callback_cls closure for @a callback
- * @return non-NULL if the notify callback was queued; can be used
- *           to cancel the request using
- *           #GNUNET_SERVER_notify_transmit_ready_cancel.
- *         NULL if we are already going to notify someone else (busy)
- */
-struct GNUNET_SERVER_TransmitHandle *
-GNUNET_SERVER_notify_transmit_ready (struct GNUNET_SERVER_Client *client,
-                                     size_t size,
-                                     struct GNUNET_TIME_Relative timeout,
-                                     GNUNET_CONNECTION_TransmitReadyNotify callback,
-                                     void *callback_cls);
-
-
-/**
- * Abort transmission request.
- *
- * @param th request to abort
- */
-void
-GNUNET_SERVER_notify_transmit_ready_cancel (struct GNUNET_SERVER_TransmitHandle *th);
-
-
-/**
- * Set the 'monitor' flag on this client.  Clients which have been
- * marked as 'monitors' won't prevent the server from shutting down
- * once #GNUNET_SERVER_stop_listening has been invoked.  The idea is
- * that for "normal" clients we likely want to allow them to process
- * their requests; however, monitor-clients are likely to 'never'
- * disconnect during shutdown and thus will not be considered when
- * determining if the server should continue to exist after
- * #GNUNET_SERVER_destroy has been called.
- *
- * @param client the client to set the 'monitor' flag on
- */
-void
-GNUNET_SERVER_client_mark_monitor (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Set the persistent flag on this client, used to setup client
- * connection to only be killed when the process of the service it's
- * connected to is actually dead.  This API is used during shutdown
- * signalling within ARM, and it is not expected that typical users
- * of the API would need this function.
- *
- * @param client the client to set the persistent flag on
- */
-void
-GNUNET_SERVER_client_persist_ (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Resume receiving from this client, we are done processing the
- * current request.  This function must be called from within each
- * #GNUNET_SERVER_MessageCallback (or its respective continuations).
- *
- * @param client client we were processing a message of
- * @param success #GNUNET_OK to keep the connection open and
- *                          continue to receive
- *                #GNUNET_NO to close the connection (normal behavior)
- *                #GNUNET_SYSERR to close the connection (signal
- *                          serious error)
- */
-void
-GNUNET_SERVER_receive_done (struct GNUNET_SERVER_Client *client,
-                            int success);
-
-
-/**
- * Change the timeout for a particular client.  Decreasing the timeout
- * may not go into effect immediately (only after the previous timeout
- * times out or activity happens on the socket).
- *
- * @param client the client to update
- * @param timeout new timeout for activities on the socket
- */
-void
-GNUNET_SERVER_client_set_timeout (struct GNUNET_SERVER_Client *client,
-                                  struct GNUNET_TIME_Relative timeout);
-
-
-/**
- * Return user context associated with the given client.
- * Note: you should probably use the macro (call without the underscore).
- *
- * @param client client to query
- * @param size number of bytes in user context struct (for verification only)
- * @return pointer to user context
- */
-void *
-GNUNET_SERVER_client_get_user_context_ (struct GNUNET_SERVER_Client *client,
-                                        size_t size);
-
-
-/**
- * Set user context to be associated with the given client.
- * Note: you should probably use the macro (call without the underscore).
- *
- * @param client client to query
- * @param ptr pointer to user context
- * @param size number of bytes in user context struct (for verification only)
- */
-void
-GNUNET_SERVER_client_set_user_context_ (struct GNUNET_SERVER_Client *client,
-                                        void *ptr,
-                                        size_t size);
-
-
-/**
- * Return user context associated with the given client.
- *
- * @param client client to query
- * @param type expected return type (i.e. 'struct Foo')
- * @return pointer to user context of type 'type *'.
- */
-#define GNUNET_SERVER_client_get_user_context(client,type)              \
-  (type *) GNUNET_SERVER_client_get_user_context_ (client, sizeof (type))
-
-/**
- * Set user context to be associated with the given client.
- *
- * @param client client to query
- * @param value pointer to user context
- */
-#define GNUNET_SERVER_client_set_user_context(client,value)             \
-  GNUNET_SERVER_client_set_user_context_ (client, value, sizeof (*value))
-
-
-/**
- * Disable the warning the server issues if a message is not acknowledged
- * in a timely fashion.  Use this call if a client is intentionally delayed
- * for a while.  Only applies to the current message.
- *
- * @param client client for which to disable the warning
- */
-void
-GNUNET_SERVER_disable_receive_done_warning (struct GNUNET_SERVER_Client
-                                            *client);
-
-
-/**
- * Inject a message into the server, pretend it came
- * from the specified client.  Delivery of the message
- * will happen instantly (if a handler is installed;
- * otherwise the call does nothing).
- *
- * @param server the server receiving the message
- * @param sender the "pretended" sender of the message
- *        can be NULL!
- * @param message message to transmit
- * @return #GNUNET_OK if the message was OK and the
- *                   connection can stay open
- *         #GNUNET_SYSERR if the connection to the
- *         client should be shut down
- */
-int
-GNUNET_SERVER_inject (struct GNUNET_SERVER_Handle *server,
-                      struct GNUNET_SERVER_Client *sender,
-                      const struct GNUNET_MessageHeader *message);
-
-
-/**
- * Add a TCP socket-based connection to the set of handles managed by
- * this server.  Use this function for outgoing (P2P) connections that
- * we initiated (and where this server should process incoming
- * messages).
- *
- * @param server the server to use
- * @param connection the connection to manage (client must
- *        stop using this connection from now on)
- * @return the client handle
- */
-struct GNUNET_SERVER_Client *
-GNUNET_SERVER_connect_socket (struct GNUNET_SERVER_Handle *server,
-                              struct GNUNET_CONNECTION_Handle *connection);
-
-
-/**
- * Notify the server that the given client handle should
- * be kept (keeps the connection up if possible, increments
- * the internal reference counter).
- *
- * @param client the client to keep
- */
-void
-GNUNET_SERVER_client_keep (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Notify the server that the given client handle is no
- * longer required.  Decrements the reference counter.  If
- * that counter reaches zero an inactive connection maybe
- * closed.
- *
- * @param client the client to drop
- */
-void
-GNUNET_SERVER_client_drop (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Obtain the network address of the other party.
- *
- * @param client the client to get the address for
- * @param addr where to store the address
- * @param addrlen where to store the length of @a addr
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_SERVER_client_get_address (struct GNUNET_SERVER_Client *client,
-                                  void **addr, size_t *addrlen);
-
-
-/**
- * Functions with this signature are called whenever a client
- * is disconnected on the network level.
- *
- * @param cls closure
- * @param client identification of the client; NULL
- *        for the last call when the server is destroyed
- */
-typedef void
-(*GNUNET_SERVER_DisconnectCallback) (void *cls,
-                                     struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Functions with this signature are called whenever a client
- * is connected on the network level.
- *
- * @param cls closure
- * @param client identification of the client
- */
-typedef void
-(*GNUNET_SERVER_ConnectCallback) (void *cls,
-                                  struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Ask the server to notify us whenever a client disconnects.
- * This function is called whenever the actual network connection
- * is closed; the reference count may be zero or larger than zero
- * at this point.  If the server is destroyed before this
- * notification is explicitly cancelled, the 'callback' will
- * once be called with a 'client' argument of NULL to indicate
- * that the server itself is now gone (and that the callback
- * won't be called anymore and also can no longer be cancelled).
- *
- * @param server the server manageing the clients
- * @param callback function to call on disconnect
- * @param callback_cls closure for @a callback
- */
-void
-GNUNET_SERVER_disconnect_notify (struct GNUNET_SERVER_Handle *server,
-                                 GNUNET_SERVER_DisconnectCallback callback,
-                                 void *callback_cls);
-
-
-/**
- * Ask the server to notify us whenever a client connects.
- * This function is called whenever the actual network connection
- * is opened. If the server is destroyed before this
- * notification is explicitly cancelled, the @a callback will
- * once be called with a 'client' argument of NULL to indicate
- * that the server itself is now gone (and that the callback
- * won't be called anymore and also can no longer be cancelled).
- *
- * @param server the server manageing the clients
- * @param callback function to call on sconnect
- * @param callback_cls closure for @a callback
- */
-void
-GNUNET_SERVER_connect_notify (struct GNUNET_SERVER_Handle *server,
-                              GNUNET_SERVER_ConnectCallback callback,
-                              void *callback_cls);
-
-
-/**
- * Ask the server to stop notifying us whenever a client disconnects.
- * Arguments must match exactly those given to
- * #GNUNET_SERVER_disconnect_notify.  It is not necessary to call this
- * function during shutdown of the server; in fact, most applications
- * will never use this function.
- *
- * @param server the server manageing the clients
- * @param callback function to call on disconnect
- * @param callback_cls closure for @a callback
- */
-void
-GNUNET_SERVER_disconnect_notify_cancel (struct GNUNET_SERVER_Handle *server,
-                                        GNUNET_SERVER_DisconnectCallback callback,
-                                        void *callback_cls);
-
-
-/**
- * Ask the server to stop notifying us whenever a client connects.
- * Arguments must match exactly those given to
- * #GNUNET_SERVER_connect_notify.  It is not necessary to call this
- * function during shutdown of the server; in fact, most applications
- * will never use this function.
- *
- * @param server the server manageing the clients
- * @param callback function to call on connect
- * @param callback_cls closure for @a callback
- */
-void
-GNUNET_SERVER_connect_notify_cancel (struct GNUNET_SERVER_Handle *server,
-                                     GNUNET_SERVER_ConnectCallback callback,
-                                     void *callback_cls);
-
-
-/**
- * Ask the server to disconnect from the given client.  This is the
- * same as passing #GNUNET_SYSERR to #GNUNET_SERVER_receive_done,
- * except that it allows dropping of a client even when not handling a
- * message from that client.
- *
- * @param client the client to disconnect from
- */
-void
-GNUNET_SERVER_client_disconnect (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Disable the "CORK" feature for communication with the given client,
- * forcing the OS to immediately flush the buffer on transmission
- * instead of potentially buffering multiple messages.
- *
- * @param client handle to the client
- * @return #GNUNET_OK on success
- */
-int
-GNUNET_SERVER_client_disable_corking (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * The tansmit context is the key datastructure for a conveniance API
- * used for transmission of complex results to the client followed
- * ONLY by signaling receive_done with success or error
- */
-struct GNUNET_SERVER_TransmitContext;
-
-
-/**
- * Create a new transmission context for the given client.
- *
- * @param client client to create the context for.
- * @return NULL on error
- */
-struct GNUNET_SERVER_TransmitContext *
-GNUNET_SERVER_transmit_context_create (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Append a message to the transmission context.
- * All messages in the context will be sent by
- * the #GNUNET_SERVER_transmit_context_run method.
- *
- * @param tc context to use
- * @param data what to append to the result message
- * @param length length of @a data
- * @param type type of the message
- */
-void
-GNUNET_SERVER_transmit_context_append_data (struct GNUNET_SERVER_TransmitContext *tc,
-                                            const void *data,
-                                            size_t length, uint16_t type);
-
-
-/**
- * Append a message to the transmission context.
- * All messages in the context will be sent by
- * the transmit_context_run method.
- *
- * @param tc context to use
- * @param msg message to append
- */
-void
-GNUNET_SERVER_transmit_context_append_message (struct GNUNET_SERVER_TransmitContext *tc,
-                                               const struct GNUNET_MessageHeader *msg);
-
-
-/**
- * Execute a transmission context.  If there is an error in the
- * transmission, the receive_done method will be called with an error
- * code (#GNUNET_SYSERR), otherwise with #GNUNET_OK.
- *
- * @param tc transmission context to use
- * @param timeout when to time out and abort the transmission
- */
-void
-GNUNET_SERVER_transmit_context_run (struct GNUNET_SERVER_TransmitContext *tc,
-                                    struct GNUNET_TIME_Relative timeout);
-
-
-/**
- * Destroy a transmission context.  This function must not be called
- * after #GNUNET_SERVER_transmit_context_run.
- *
- * @param tc transmission context to destroy
- * @param success code to give to #GNUNET_SERVER_receive_done  for
- *        the client: #GNUNET_OK to keep the connection open and
- *                          continue to receive
- *                #GNUNET_NO to close the connection (normal behavior)
- *                #GNUNET_SYSERR to close the connection (signal
- *                          serious error)
- */
-void
-GNUNET_SERVER_transmit_context_destroy (struct GNUNET_SERVER_TransmitContext *tc,
-                                        int success);
-
-
-/**
- * The notification context is the key datastructure for a conveniance
- * API used for transmission of notifications to the client until the
- * client disconnects or is disconnected (or the notification context
- * is destroyed, in which case we disconnect these clients).
- * Essentially, all (notification) messages are queued up until the
- * client is able to read them.
- */
-struct GNUNET_SERVER_NotificationContext;
-
-
-/**
- * Create a new notification context.
- *
- * @param server server for which this function creates the context
- * @param queue_length maximum number of messages to keep in
- *        the notification queue; optional messages are dropped
- *        if the queue gets longer than this number of messages
- * @return handle to the notification context
- */
-struct GNUNET_SERVER_NotificationContext *
-GNUNET_SERVER_notification_context_create (struct GNUNET_SERVER_Handle *server,
-                                           unsigned int queue_length);
-
-
-/**
- * Destroy the context, force disconnect for all clients.
- *
- * @param nc context to destroy.
- */
-void
-GNUNET_SERVER_notification_context_destroy (struct GNUNET_SERVER_NotificationContext *nc);
-
-
-/**
- * Add a client to the notification context.
- *
- * @param nc context to modify
- * @param client client to add
- */
-void
-GNUNET_SERVER_notification_context_add (struct GNUNET_SERVER_NotificationContext *nc,
-                                        struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Send a message to a particular client; must have
- * already been added to the notification context.
- *
- * @param nc context to modify
- * @param client client to transmit to
- * @param msg message to send
- * @param can_drop can this message be dropped due to queue length limitations
- */
-void
-GNUNET_SERVER_notification_context_unicast (struct GNUNET_SERVER_NotificationContext *nc,
-                                            struct GNUNET_SERVER_Client *client,
-                                            const struct GNUNET_MessageHeader *msg,
-                                            int can_drop);
-
-
-/**
- * Send a message to all clients of this context.
- *
- * @param nc context to modify
- * @param msg message to send
- * @param can_drop can this message be dropped due to queue length limitations
- */
-void
-GNUNET_SERVER_notification_context_broadcast (struct GNUNET_SERVER_NotificationContext *nc,
-                                              const struct GNUNET_MessageHeader *msg,
-                                              int can_drop);
-
-
-/**
- * Return active number of subscribers in this context.
- *
- * @param nc context to query
- * @return number of current subscribers
- */
-unsigned int
-GNUNET_SERVER_notification_context_get_size (struct GNUNET_SERVER_NotificationContext *nc);
-
-
-/**
- * Create a message queue for a server's client.
- *
- * @param client the client
- * @return the message queue
- */
-struct GNUNET_MQ_Handle *
-GNUNET_MQ_queue_for_server_client (struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Handle to a message stream tokenizer.
- */
-struct GNUNET_SERVER_MessageStreamTokenizer;
-
-
-/**
- * Functions with this signature are called whenever a
- * complete message is received by the tokenizer.
- *
- * Do not call #GNUNET_SERVER_mst_destroy from within
- * the scope of this callback.
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
- */
-typedef int
-(*GNUNET_SERVER_MessageTokenizerCallback) (void *cls,
-                                           void *client,
-                                           const struct GNUNET_MessageHeader *message);
-
-
-/**
- * Create a message stream tokenizer.
- *
- * @param cb function to call on completed messages
- * @param cb_cls closure for @a cb
- * @return handle to tokenizer
- */
-struct GNUNET_SERVER_MessageStreamTokenizer *
-GNUNET_SERVER_mst_create (GNUNET_SERVER_MessageTokenizerCallback cb,
-                          void *cb_cls);
-
-
-/**
- * Add incoming data to the receive buffer and call the
- * callback for all complete messages.
- *
- * @param mst tokenizer to use
- * @param client_identity ID of client for which this is a buffer,
- *        can be NULL (will be passed back to 'cb')
- * @param buf input data to add
- * @param size number of bytes in @a buf
- * @param purge should any excess bytes in the buffer be discarded
- *       (i.e. for packet-based services like UDP)
- * @param one_shot only call callback once, keep rest of message in buffer
- * @return #GNUNET_OK if we are done processing (need more data)
- *         #GNUNET_NO if one_shot was set and we have another message ready
- *         #GNUNET_SYSERR if the data stream is corrupt
- */
-int
-GNUNET_SERVER_mst_receive (struct GNUNET_SERVER_MessageStreamTokenizer *mst,
-                           void *client_identity,
-                           const char *buf, size_t size,
-                           int purge, int one_shot);
-
-
-/**
- * Destroys a tokenizer.
- *
- * @param mst tokenizer to destroy
- */
-void
-GNUNET_SERVER_mst_destroy (struct GNUNET_SERVER_MessageStreamTokenizer *mst);
-
-
-/**
- * Signature of a function to create a custom tokenizer.
- *
- * @param cls closure from #GNUNET_SERVER_set_callbacks
- * @param client handle to client the tokenzier will be used for
- * @return handle to custom tokenizer ('mst')
- */
-typedef void*
-(*GNUNET_SERVER_MstCreateCallback) (void *cls,
-                                    struct GNUNET_SERVER_Client *client);
-
-
-/**
- * Signature of a function to destroy a custom tokenizer.
- *
- * @param cls closure from #GNUNET_SERVER_set_callbacks
- * @param mst custom tokenizer handle
- */
-typedef void
-(*GNUNET_SERVER_MstDestroyCallback) (void *cls,
-                                     void *mst);
-
-
-/**
- * Signature of a function to receive data for a custom tokenizer.
- *
- * @param cls closure from #GNUNET_SERVER_set_callbacks
- * @param mst custom tokenizer handle
- * @param client_identity ID of client for which this is a buffer,
- *        can be NULL (will be passed back to 'cb')
- * @param buf input data to add
- * @param size number of bytes in @a buf
- * @param purge should any excess bytes in the buffer be discarded
- *       (i.e. for packet-based services like UDP)
- * @param one_shot only call callback once, keep rest of message in buffer
- * @return #GNUNET_OK if we are done processing (need more data)
- *         #GNUNET_NO if one_shot was set and we have another message ready
- *         #GNUNET_SYSERR if the data stream is corrupt
- */
-typedef int
-(*GNUNET_SERVER_MstReceiveCallback) (void *cls, void *mst,
-                                     struct GNUNET_SERVER_Client *client,
-                                     const char *buf,
-                                     size_t size,
-                                     int purge,
-                                     int one_shot);
-
-
-/**
- * Change functions used by the server to tokenize the message stream.
- * (very rarely used).
- *
- * @param server server to modify
- * @param create new tokenizer initialization function
- * @param destroy new tokenizer destruction function
- * @param receive new tokenizer receive function
- * @param cls closure for @a create, @a receive and @a destroy
- */
-void
-GNUNET_SERVER_set_callbacks (struct GNUNET_SERVER_Handle *server,
-                             GNUNET_SERVER_MstCreateCallback create,
-                             GNUNET_SERVER_MstDestroyCallback destroy,
-                             GNUNET_SERVER_MstReceiveCallback receive,
-                             void *cls);
-
-
-#if 0                           /* keep Emacsens' auto-indent happy */
-{
-#endif
-#ifdef __cplusplus
-}
-#endif
-
-/* ifndef GNUNET_SERVER_LIB_H */
-#endif
-
-/** @} */  /* end of group server */
-
-/* end of gnunet_server_lib.h */
diff --git a/src/include/gnunet_service_lib.h b/src/include/gnunet_service_lib.h
index 75b880530..aacafe956 100644
--- a/src/include/gnunet_service_lib.h
+++ b/src/include/gnunet_service_lib.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009-2013, 2016 GNUnet e.V.
+     Copyright (C) 2009-2013, 2016, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -44,25 +44,10 @@ extern "C"
 #endif
 
 #include "gnunet_configuration_lib.h"
-#include "gnunet_server_lib.h"
 #include "gnunet_mq_lib.h"
 
 
 /**
- * Function called by the service's run
- * method to run service-specific setup code.
- *
- * @param cls closure
- * @param server the initialized server
- * @param cfg configuration to use
- */
-typedef void
-(*GNUNET_SERVICE_Main) (void *cls,
-                        struct GNUNET_SERVER_Handle *server,
-                        const struct GNUNET_CONFIGURATION_Handle *cfg);
-
-
-/**
  * Options for the service (bitmask).
  */
 enum GNUNET_SERVICE_Options
@@ -88,84 +73,6 @@ enum GNUNET_SERVICE_Options
 };
 
 
-/**
- * Run a standard GNUnet service startup sequence (initialize loggers
- * and configuration, parse options).
- *
- * @param argc number of command line arguments in @a argv
- * @param argv command line arguments
- * @param service_name our service name
- * @param options service options
- * @param task main task of the service
- * @param task_cls closure for @a task
- * @return #GNUNET_SYSERR on error, #GNUNET_OK
- *         if we shutdown nicely
- * @deprecated
- */
-int
-GNUNET_SERVICE_run (int argc,
-                    char *const *argv,
-                    const char *service_name,
-                    enum GNUNET_SERVICE_Options options,
-                    GNUNET_SERVICE_Main task,
-                    void *task_cls);
-
-
-/**
- * Opaque handle for a service.
- */
-struct GNUNET_SERVICE_Context;
-
-
-/**
- * Run a service startup sequence within an existing
- * initialized system.
- *
- * @param service_name our service name
- * @param cfg configuration to use
- * @param options service options
- * @return NULL on error, service handle
- * @deprecated
- */
-struct GNUNET_SERVICE_Context *
-GNUNET_SERVICE_start (const char *service_name,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      enum GNUNET_SERVICE_Options options);
-
-
-/**
- * Obtain the server used by a service.  Note that the server must NOT
- * be destroyed by the caller.
- *
- * @param ctx the service context returned from the start function
- * @return handle to the server for this service, NULL if there is none
- * @deprecated
- */
-struct GNUNET_SERVER_Handle *
-GNUNET_SERVICE_get_server (struct GNUNET_SERVICE_Context *ctx);
-
-
-/**
- * Get the NULL-terminated array of listen sockets for this service.
- *
- * @param ctx service context to query
- * @return NULL if there are no listen sockets, otherwise NULL-terminated
- *              array of listen sockets.
- * @deprecated
- */
-struct GNUNET_NETWORK_Handle *const *
-GNUNET_SERVICE_get_listen_sockets (struct GNUNET_SERVICE_Context *ctx);
-
-
-/**
- * Stop a service that was started with #GNUNET_SERVICE_start.
- *
- * @param sctx the service context returned from the start function
- * @deprecated
- */
-void
-GNUNET_SERVICE_stop (struct GNUNET_SERVICE_Context *sctx);
-
 
 /* **************** NEW SERVICE API ********************** */
 
@@ -245,7 +152,7 @@ typedef void
  * dropped. Additionally, clients can be dropped at any time using
  * #GNUNET_SERVICE_client_drop().
  *
- * The service must be stopped using #GNUNET_SERVICE_stoP().
+ * The service must be stopped using #GNUNET_SERVICE_stop().
  *
  * @param service_name name of the service to run
  * @param cfg configuration to use
@@ -258,7 +165,7 @@ typedef void
  * @return NULL on error
  */
 struct GNUNET_SERVICE_Handle *
-GNUNET_SERVICE_starT (const char *service_name,
+GNUNET_SERVICE_start (const char *service_name,
                       const struct GNUNET_CONFIGURATION_Handle *cfg,
                       GNUNET_SERVICE_ConnectHandler connect_cb,
                       GNUNET_SERVICE_DisconnectHandler disconnect_cb,
@@ -267,12 +174,12 @@ GNUNET_SERVICE_starT (const char *service_name,
 
 
 /**
- * Stops a service that was started with #GNUNET_SERVICE_starT().
+ * Stops a service that was started with #GNUNET_SERVICE_start().
  *
  * @param srv service to stop
  */
 void
-GNUNET_SERVICE_stoP (struct GNUNET_SERVICE_Handle *srv);
+GNUNET_SERVICE_stop (struct GNUNET_SERVICE_Handle *srv);
 
 
 /**
@@ -317,7 +224,7 @@ GNUNET_SERVICE_stoP (struct GNUNET_SERVICE_Handle *srv);
  * @return 0 on success, non-zero on error
  */
 int
-GNUNET_SERVICE_ruN_ (int argc,
+GNUNET_SERVICE_run_ (int argc,
                      char *const *argv,
                      const char *service_name,
                      enum GNUNET_SERVICE_Options options,
@@ -393,7 +300,7 @@ GNUNET_SERVICE_ruN_ (int argc,
     struct GNUNET_MQ_MessageHandler mh[] = { \
       __VA_ARGS__ \
     };                        \
-    return GNUNET_SERVICE_ruN_ (argc, \
+    return GNUNET_SERVICE_run_ (argc, \
                                 argv, \
                                 service_name, \
                                 service_options, \
diff --git a/src/include/gnunet_sq_lib.h b/src/include/gnunet_sq_lib.h
index c03f83e07..c196d7767 100644
--- a/src/include/gnunet_sq_lib.h
+++ b/src/include/gnunet_sq_lib.h
@@ -182,6 +182,29 @@ GNUNET_SQ_query_param_uint64 (const uint64_t *x);
 
 
 /**
+ * Execute binding operations for a prepared statement.
+ *
+ * @param db_conn database connection
+ * @param params parameters to the statement
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+int
+GNUNET_SQ_bind (sqlite3_stmt *stmt,
+                const struct GNUNET_SQ_QueryParam *params);
+
+
+/**
+ * Reset @a stmt and log error.
+ *
+ * @param dbh database handle
+ * @param stmt statement to reset
+ */
+void
+GNUNET_SQ_reset (sqlite3 *dbh,
+                 sqlite3_stmt *stmt);
+
+
+/**
  * Extract data from a Postgres database @a result at row @a row.
  *
  * @param cls closure
@@ -400,18 +423,6 @@ GNUNET_SQ_result_spec_uint64 (uint64_t *u64);
 
 
 /**
- * Execute a prepared statement.
- *
- * @param db_conn database connection
- * @param params parameters to the statement
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-int
-GNUNET_SQ_bind (sqlite3_stmt *stmt,
-                const struct GNUNET_SQ_QueryParam *params);
-
-
-/**
  * Extract results from a query result according to the given specification.
  *
  * @param result result to process
diff --git a/src/include/gnunet_util_lib.h b/src/include/gnunet_util_lib.h
index 52f5d8ab2..a9bd6c33f 100644
--- a/src/include/gnunet_util_lib.h
+++ b/src/include/gnunet_util_lib.h
@@ -38,10 +38,24 @@ extern "C"
 #endif
 #endif
 
+
+/**
+ * Largest supported message (to be precise, one byte more
+ * than the largest possible message, so tests involving
+ * this value should check for messages being smaller than
+ * this value).
+ */
+#define GNUNET_MAX_MESSAGE_SIZE 65536
+
+/**
+ * Smallest supported message.
+ */
+#define GNUNET_MIN_MESSAGE_SIZE sizeof (struct GNUNET_MessageHeader)
+
+
 #include "gnunet_crypto_lib.h"
 #include "gnunet_bandwidth_lib.h"
 #include "gnunet_bio_lib.h"
-#include "gnunet_connection_lib.h"
 #include "gnunet_client_lib.h"
 #include "gnunet_container_lib.h"
 #include "gnunet_getopt_lib.h"
@@ -55,7 +69,6 @@ extern "C"
 #include "gnunet_plugin_lib.h"
 #include "gnunet_program_lib.h"
 #include "gnunet_protocols.h"
-#include "gnunet_server_lib.h"
 #include "gnunet_service_lib.h"
 #include "gnunet_signal_lib.h"
 #include "gnunet_strings_lib.h"
diff --git a/src/include/platform.h b/src/include/platform.h
index add58821f..6095d0258 100644
--- a/src/include/platform.h
+++ b/src/include/platform.h
@@ -110,6 +110,7 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <stdarg.h>
+#include <stdbool.h>
 #include <errno.h>
 #include <signal.h>
 #include <libgen.h>
diff --git a/src/integration-tests/confs/test_defaults.conf b/src/integration-tests/confs/test_defaults.conf
index 8d3356cbc..1be582650 100644
--- a/src/integration-tests/confs/test_defaults.conf
+++ b/src/integration-tests/confs/test_defaults.conf
@@ -1,5 +1,5 @@
-@INLINE@ ../../contrib/no_forcestart.conf
-@INLINE@ ../../contrib/no_autostart_above_core.conf
+@INLINE@ ../../../contrib/no_forcestart.conf
+@INLINE@ ../../../contrib/no_autostart_above_core.conf
 
 [fs]
 FORCESTART = YES
diff --git a/src/json/json.c b/src/json/json.c
index a2d1a9608..c182a02f4 100644
--- a/src/json/json.c
+++ b/src/json/json.c
@@ -1,6 +1,6 @@
 /*
   This file is part of GNUnet
-  Copyright (C) 2014, 2015, 2016 GNUnet e.V.
+  Copyright (C) 2014-2017 GNUnet e.V.
 
   GNUnet is free software; you can redistribute it and/or modify it under the
   terms of the GNU General Public License as published by the Free Software
@@ -82,13 +82,78 @@ GNUNET_JSON_parse (const json_t *root,
 void
 GNUNET_JSON_parse_free (struct GNUNET_JSON_Specification *spec)
 {
-  unsigned int i;
-
-  for (i=0;NULL != spec[i].parser;i++)
+  for (unsigned int i=0;NULL != spec[i].parser;i++)
     if (NULL != spec[i].cleaner)
       spec[i].cleaner (spec[i].cls,
                        &spec[i]);
 }
 
 
+/**
+ * Set an option with a JSON value from the command line.
+ * A pointer to this function should be passed as part of the
+ * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
+ * of this type.
+ *
+ * @param ctx command line processing context
+ * @param scls additional closure (will point to the 'json_t *')
+ * @param option name of the option
+ * @param value actual value of the option as a string.
+ * @return #GNUNET_OK if parsing the value worked
+ */
+static int
+set_json (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+          void *scls,
+          const char *option,
+          const char *value)
+{
+  json_t **json = scls;
+  json_error_t error;
+
+  *json = json_loads (value,
+                      JSON_REJECT_DUPLICATES,
+                      &error);
+  if (NULL == *json)
+  {
+    FPRINTF (stderr,
+             _("Failed to parse JSON in option `%s': %s (%s)\n"),
+             option,
+             error.text,
+             error.source);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Allow user to specify a JSON input value.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the JSON specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_JSON_getopt (char shortName,
+                    const char *name,
+                    const char *argumentHelp,
+                    const char *description,
+                    json_t **json)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_json,
+    .scls = (void *) json
+  };
+
+  return clo;
+}
+
+
 /* end of json.c */
diff --git a/src/multicast/.gitignore b/src/multicast/.gitignore
index d38ca9c02..43752ec4b 100644
--- a/src/multicast/.gitignore
+++ b/src/multicast/.gitignore
@@ -2,3 +2,4 @@ gnunet-service-multicast
 gnunet-multicast
 test_multicast 
 test_multicast_multipeer
+test_multicast_2peers
diff --git a/src/multicast/Makefile.am b/src/multicast/Makefile.am
index 61821d973..1a61abb09 100644
--- a/src/multicast/Makefile.am
+++ b/src/multicast/Makefile.am
@@ -33,7 +33,8 @@ bin_PROGRAMS = \
  gnunet-multicast
 
 libexec_PROGRAMS = \
- gnunet-service-multicast
+ gnunet-service-multicast \
+ $(EXP_LIBEXEC)
 
 gnunet_multicast_SOURCES = \
  gnunet-multicast.c
@@ -45,17 +46,17 @@ gnunet_service_multicast_SOURCES = \
  gnunet-service-multicast.c
 gnunet_service_multicast_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(GN_LIBINTL)
 
-
 check_PROGRAMS = \
  test_multicast \
- test_multicast_multipeer
+ test_multicast_2peers
+# test_multicast_multipeer
 
 if ENABLE_TEST_RUN
-AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@}; export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH; unset XDG_DATA_HOME; unset XDG_CONFIG_HOME; export GNUNET_FORCE_LOG=';;;;INFO'
+AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@}; export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH; unset XDG_DATA_HOME; unset XDG_CONFIG_HOME; export GNUNET_FORCE_LOG=';;;;INFO';
 TESTS = $(check_PROGRAMS)
 endif
 
@@ -66,9 +67,9 @@ test_multicast_LDADD = \
   $(top_builddir)/src/testing/libgnunettesting.la \
   $(top_builddir)/src/util/libgnunetutil.la
 
-test_multicast_multipeer_SOURCE = \
- test_multicast_multipeer.c
-test_multicast_multipeer_LDADD = \
+test_multicast_2peers_SOURCE = \
+ test_multicast_2peers.c
+test_multicast_2peers_LDADD = \
   libgnunetmulticast.la \
   $(top_builddir)/src/testbed/libgnunettestbed.la \
   $(top_builddir)/src/util/libgnunetutil.la
diff --git a/src/multicast/gnunet-service-multicast.c b/src/multicast/gnunet-service-multicast.c
index 3f356af83..9683efcff 100644
--- a/src/multicast/gnunet-service-multicast.c
+++ b/src/multicast/gnunet-service-multicast.c
@@ -1302,7 +1302,7 @@ cadet_channel_create (struct Group *grp, struct GNUNET_PeerIdentity *peer)
     GNUNET_MQ_handler_end ()
   };
 
-  chn->channel = GNUNET_CADET_channel_creatE (cadet, chn, &chn->peer,
+  chn->channel = GNUNET_CADET_channel_create (cadet, chn, &chn->peer,
                                               &grp->cadet_port_hash,
                                               GNUNET_CADET_OPTION_RELIABLE,
                                               cadet_notify_window_change,
@@ -1392,7 +1392,7 @@ handle_client_origin_start (void *cls,
     };
 
 
-    orig->cadet_port = GNUNET_CADET_open_porT (cadet,
+    orig->cadet_port = GNUNET_CADET_open_port (cadet,
                                                &grp->cadet_port_hash,
                                                cadet_notify_connect,
                                                NULL,
@@ -2085,7 +2085,7 @@ run (void *cls,
   replay_req_cadet = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
   replay_req_client = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_NO);
 
-  cadet = GNUNET_CADET_connecT (cfg);
+  cadet = GNUNET_CADET_connect (cfg);
 
   GNUNET_assert (NULL != cadet);
 
diff --git a/src/multicast/test_multicast.conf b/src/multicast/test_multicast.conf
index 675776bbc..3081aeecc 100644
--- a/src/multicast/test_multicast.conf
+++ b/src/multicast/test_multicast.conf
@@ -1,4 +1,12 @@
+[testbed]
+HOSTNAME = localhost
+
 [arm]
 GLOBAL_POSTFIX=-L ERROR
 
-#PREFIX = sakura -t test-multicast -e cgdb --args
+[multicast]
+#PREFIX = xterm -T peer -e gdb --args
+UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-multicast.sock
+
+[vpn]
+AUTOSTART = NO
diff --git a/src/multicast/test_multicast_2peers.c b/src/multicast/test_multicast_2peers.c
new file mode 100644
index 000000000..058533e50
--- /dev/null
+++ b/src/multicast/test_multicast_2peers.c
@@ -0,0 +1,511 @@
+/*
+ * This file is part of GNUnet
+ * Copyright (C) 2013 GNUnet e.V.
+ *
+ * GNUnet is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; either version 3, or (at your
+ * option) any later version.
+ *
+ * GNUnet is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with GNUnet; see the file COPYING.  If not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file multicast/test_multicast_2peers.c
+ * @brief Tests for the Multicast API with two peers doing the ping
+ *        pong test.
+ * @author xrs
+ */
+
+#include <inttypes.h>
+
+#include "platform.h"
+#include "gnunet_crypto_lib.h"
+#include "gnunet_common.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_testbed_service.h"
+#include "gnunet_multicast_service.h"
+
+#define NUM_PEERS 2
+
+static struct GNUNET_TESTBED_Operation *op0;
+static struct GNUNET_TESTBED_Operation *op1;
+static struct GNUNET_TESTBED_Operation *pi_op0;
+static struct GNUNET_TESTBED_Operation *pi_op1;
+
+static struct GNUNET_TESTBED_Peer **peers;
+const struct GNUNET_PeerIdentity *peer_id[2];
+
+static struct GNUNET_SCHEDULER_Task *timeout_tid;
+
+static struct GNUNET_MULTICAST_Origin *origin;
+static struct GNUNET_MULTICAST_Member *member;
+
+struct GNUNET_CRYPTO_EddsaPrivateKey *group_key;
+struct GNUNET_CRYPTO_EddsaPublicKey group_pub_key;
+
+struct GNUNET_CRYPTO_EcdsaPrivateKey *member_key;
+struct GNUNET_CRYPTO_EcdsaPublicKey member_pub_key;
+
+/**
+ * Global result for testcase.
+ */
+static int result;
+
+
+/**
+ * Function run on CTRL-C or shutdown (i.e. success/timeout/etc.).
+ * Cleans up.
+ */
+static void
+shutdown_task (void *cls)
+{
+  if (NULL != op0)
+  {
+    GNUNET_TESTBED_operation_done (op0);
+    op0 = NULL;
+  }
+  if (NULL != op1)
+  {
+    GNUNET_TESTBED_operation_done (op1);
+    op1 = NULL;
+  }
+  if (NULL != pi_op0)
+  {
+    GNUNET_TESTBED_operation_done (pi_op0);
+    pi_op0 = NULL;
+  }
+  if (NULL != pi_op1)
+  {
+    GNUNET_TESTBED_operation_done (pi_op1);
+    pi_op1 = NULL;
+  }
+  if (NULL != timeout_tid)
+    {
+      GNUNET_SCHEDULER_cancel (timeout_tid);
+      timeout_tid = NULL;
+    }
+}
+
+
+static void
+timeout_task (void *cls)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+              "Timeout!\n");
+  result = GNUNET_SYSERR;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+static void
+member_join_request (void *cls,
+                     const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
+                     const struct GNUNET_MessageHeader *join_msg,
+                     struct GNUNET_MULTICAST_JoinHandle *jh)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Member sent a join request.\n");
+
+}
+
+
+static int
+notify (void *cls,
+        size_t *data_size,
+        void *data)
+{
+
+  char text[] = "ping";
+  *data_size = strlen(text)+1;
+  GNUNET_memcpy(data, text, *data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Member sents message to origin: %s\n", text);
+
+  return GNUNET_YES;
+}
+
+
+static void
+member_join_decision (void *cls,
+                      int is_admitted,
+                      const struct GNUNET_PeerIdentity *peer,
+                      uint16_t relay_count,
+                      const struct GNUNET_PeerIdentity *relays,
+                      const struct GNUNET_MessageHeader *join_msg)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Member received a decision from origin: %s\n",
+              (GNUNET_YES == is_admitted)
+              ? "accepted"
+              : "rejected");
+
+  if (GNUNET_YES == is_admitted)
+  {
+    struct GNUNET_MULTICAST_MemberTransmitHandle *req;
+
+    // FIXME: move to MQ-style API!
+    req = GNUNET_MULTICAST_member_to_origin (member,
+                                             0,
+                                             &notify,
+                                             NULL);
+  }
+}
+
+
+static void
+member_message (void *cls,
+                const struct GNUNET_MULTICAST_MessageHeader *msg)
+{
+  if (0 != strncmp ("pong", (char *)&msg[1], 4))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "member did not receive pong\n");
+    result = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+  }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "member receives: %s\n", (char *)&msg[1]);
+
+  // Testcase ends here.
+  result = GNUNET_YES;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+static void
+origin_join_request (void *cls,
+                 const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
+                 const struct GNUNET_MessageHeader *join_msg,
+                 struct GNUNET_MULTICAST_JoinHandle *jh)
+{
+  struct GNUNET_MessageHeader *join_resp;
+
+  uint8_t data_size = ntohs (join_msg->size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "origin got a join request...\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "origin receives: '%s'\n", (char *)&join_msg[1]);
+
+  const char data[] = "Come in!";
+  data_size = strlen (data) + 1;
+  join_resp = GNUNET_malloc (sizeof (join_resp) + data_size);
+  join_resp->size = htons (sizeof (join_resp) + data_size);
+  join_resp->type = htons (123);
+  GNUNET_memcpy (&join_resp[1], data, data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "origin sends: '%s'\n", data);
+
+  GNUNET_MULTICAST_join_decision (jh,
+                                  GNUNET_YES,
+                                  0,
+                                  NULL,
+                                  join_resp);
+  GNUNET_free (join_resp);
+  result = GNUNET_OK;
+}
+
+int
+origin_notify (void *cls,
+               size_t *data_size,
+               void *data)
+{
+  char text[] = "pong";
+  *data_size = strlen(text)+1;
+  memcpy(data, text, *data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin sends (to all): %s\n", text);
+
+  return GNUNET_YES;
+}
+
+
+static void
+origin_request (void *cls,
+                const struct GNUNET_MULTICAST_RequestHeader *req)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin receives: %s\n", (char *)&req[1]);
+
+  if (0 != strncmp ("ping", (char *)&req[1], 4))
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "origin didn't reveice a correct request");
+
+  GNUNET_MULTICAST_origin_to_all (origin,
+                                  0,
+                                  0,
+                                  origin_notify,
+                                  NULL);
+}
+
+static void
+origin_message (void *cls,
+                const struct GNUNET_MULTICAST_MessageHeader *msg)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin message msg\n");
+}
+
+
+static void
+service_connect1 (void *cls,
+                  struct GNUNET_TESTBED_Operation *op,
+                  void *ca_result,
+                  const char *emsg)
+{
+  member = ca_result;
+
+  if (NULL != member)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Connected to multicast service of member\n");
+  }
+  else
+  {
+    result = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+  }
+}
+
+static void
+multicast_da1 (void *cls,
+               void * op_result)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Member parting from multicast group\n");
+
+  GNUNET_MULTICAST_member_part (member, NULL, NULL);
+}
+
+
+static void *
+multicast_ca1 (void *cls,
+               const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  struct GNUNET_MessageHeader *join_msg;
+  void *ret;
+
+  // Get members keys
+  member_key = GNUNET_CRYPTO_ecdsa_key_create ();
+  GNUNET_CRYPTO_ecdsa_key_get_public (member_key, &member_pub_key);
+
+  char data[] = "Hi, can I enter?";
+  uint8_t data_size = strlen (data) + 1;
+  join_msg = GNUNET_malloc (sizeof (join_msg) + data_size);
+  join_msg->size = htons (sizeof (join_msg) + data_size);
+  join_msg->type = htons (123);
+  GNUNET_memcpy (&join_msg[1], data, data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Members tries to join multicast group\n");
+
+  ret = GNUNET_MULTICAST_member_join (cfg,
+                                       &group_pub_key,
+                                       member_key,
+                                       peer_id[0],
+                                       0,
+                                       NULL,
+                                       join_msg, /* join message */
+                                       member_join_request,
+                                       member_join_decision,
+                                       NULL, /* no test for member_replay_frag */
+                                       NULL, /* no test for member_replay_msg */
+                                       member_message,
+                                       NULL);
+  GNUNET_free (join_msg);
+  return ret;
+}
+
+
+static void
+peer_information_cb (void *cls,
+                     struct GNUNET_TESTBED_Operation *op,
+                     const struct GNUNET_TESTBED_PeerInformation *pinfo,
+                     const char *emsg)
+{
+  int i = (int) (long) cls;
+
+  if (NULL == pinfo)
+  {
+    result = GNUNET_SYSERR;
+    GNUNET_SCHEDULER_shutdown ();
+  }
+
+  peer_id[i] = pinfo->result.id;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Got peer information of %s (%s)\n", (0==i)?"origin":"member" ,GNUNET_i2s(pinfo->result.id));
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Create member peer\n");
+
+  if (0 == i)
+  {
+    /* connect to multicast service of member */
+    op1 = GNUNET_TESTBED_service_connect (NULL,                    /* Closure for operation */
+                                          peers[1],                /* The peer whose service to connect to */
+                                          "multicast",             /* The name of the service */
+                                          service_connect1,   /* callback to call after a handle to service
+                                                                 is opened */
+                                          NULL,                    /* closure for the above callback */
+                                          multicast_ca1,      /* callback to call with peer's configuration;
+                                                                 this should open the needed service connection */
+                                          multicast_da1,     /* callback to be called when closing the
+                                                                opened service connection */
+                                          NULL);                   /* closure for the above two callbacks */
+    }
+}
+
+/**
+ * Test logic of peer "0" being origin starts here.
+ *
+ * @param cls closure, for the example: NULL
+ * @param op should be equal to "dht_op"
+ * @param ca_result result of the connect operation, the
+ *        connection to the DHT service
+ * @param emsg error message, if testbed somehow failed to
+ *        connect to the DHT.
+ */
+static void
+service_connect0 (void *cls,
+                  struct GNUNET_TESTBED_Operation *op,
+                  void *ca_result,
+                  const char *emsg)
+{
+  origin = ca_result;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Connected to multicast service of origin\n");
+
+  // Get GNUnet identity of origin
+  pi_op0 = GNUNET_TESTBED_peer_get_information (peers[0],
+                                               GNUNET_TESTBED_PIT_IDENTITY,
+                                               peer_information_cb,
+                                               (void *) 0);
+  // Get GNUnet identity of member
+  pi_op1 = GNUNET_TESTBED_peer_get_information (peers[1],
+                                               GNUNET_TESTBED_PIT_IDENTITY,
+                                               peer_information_cb,
+                                               (void *) 1);
+
+  /* Connection to service successful. Here we'd usually do something with
+   * the service. */
+  result = GNUNET_OK;
+  //GNUNET_SCHEDULER_shutdown (); /* Also kills the testbed */
+}
+
+
+
+/**
+ * Function run when service multicast has started and is providing us
+ * with a configuration file.
+ */
+static void *
+multicast_ca0 (void *cls,
+               const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  group_key = GNUNET_CRYPTO_eddsa_key_create ();
+  GNUNET_CRYPTO_eddsa_key_get_public (group_key, &group_pub_key);
+
+  return GNUNET_MULTICAST_origin_start (cfg,
+                                        group_key,
+                                        0,
+                                        origin_join_request,
+                                        NULL, /* no test for origin_replay_frag */
+                                        NULL, /* no test for origin_replay_msg */
+                                        origin_request,
+                                        origin_message,
+                                        NULL);
+}
+
+static void
+multicast_da0 (void *cls,
+               void *op_result)
+{
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Origin closes multicast group\n");
+
+  GNUNET_MULTICAST_origin_stop (origin, NULL, NULL);
+}
+
+
+/**
+ * Main function inovked from TESTBED once all of the
+ * peers are up and running.  This one then connects
+ * just to the multicast service of peer 0 and 1.
+ * Peer 0 is going to be origin.
+ * Peer 1 is going to be one member.
+ * Origin will start a multicast group and the member will try to join it.
+ * After that we execute some multicast test.
+ *
+ * @param cls closure
+ * @param h the run handle
+ * @param peers started peers for the test
+ * @param num_peers size of the 'peers' array
+ * @param links_succeeded number of links between peers that were created
+ * @param links_failed number of links testbed was unable to establish
+ */
+static void
+testbed_master (void *cls,
+     struct GNUNET_TESTBED_RunHandle *h,
+     unsigned int num_peers,
+     struct GNUNET_TESTBED_Peer **p,
+     unsigned int links_succeeded,
+     unsigned int links_failed)
+{
+  /* Testbed is ready with peers running and connected in a pre-defined overlay
+     topology (FIXME)  */
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Connected to testbed_master()\n");
+
+  peers = p;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Create origin peer\n");
+  op0 = GNUNET_TESTBED_service_connect (NULL,                    /* Closure for operation */
+                                        peers[0],                /* The peer whose service to connect to */
+                                        "multicast",             /* The name of the service */
+                                        service_connect0,   /* callback to call after a handle to service
+                                                               is opened */
+                                        NULL,                    /* closure for the above callback */
+                                        multicast_ca0,      /* callback to call with peer's configuration;
+                                                               this should open the needed service connection */
+                                        multicast_da0,     /* callback to be called when closing the
+                                                              opened service connection */
+                                        NULL);                   /* closure for the above two callbacks */
+
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); /* Schedule a new task on shutdown */
+
+  /* Schedule the shutdown task with a delay of a few Seconds */
+  timeout_tid = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 50),
+                                              &timeout_task, NULL);
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  int ret;
+
+  result = GNUNET_SYSERR;
+  ret = GNUNET_TESTBED_test_run
+      ("test-multicast-multipeer",  /* test case name */
+       "test_multicast.conf", /* template configuration */
+       NUM_PEERS,       /* number of peers to start */
+       0LL, /* Event mask - set to 0 for no event notifications */
+       NULL, /* Controller event callback */
+       NULL, /* Closure for controller event callback */
+       testbed_master, /* continuation callback to be called when testbed setup is complete */
+       NULL); /* Closure for the test_master callback */
+  if ( (GNUNET_OK != ret) || (GNUNET_OK != result) )
+    return 1;
+  return 0;
+}
+
+/* end of test_multicast_multipeer.c */
diff --git a/src/multicast/test_multicast_multipeer.c b/src/multicast/test_multicast_multipeer.c
index 1b76737f4..58e43d4ba 100644
--- a/src/multicast/test_multicast_multipeer.c
+++ b/src/multicast/test_multicast_multipeer.c
@@ -107,7 +107,6 @@ shutdown_task (void *cls)
 static void
 timeout_task (void *cls)
 {
-  timeout_tid = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
               "Timeout!\n");
   result = GNUNET_SYSERR;
@@ -126,6 +125,21 @@ member_join_request (void *cls,
 
 }
 
+int notify (void *cls,
+            size_t *data_size,
+            void *data)
+{
+
+  char text[] = "ping";
+  *data_size = strlen(text)+1;
+  GNUNET_memcpy(data, text, *data_size);
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+              "Member sents message to origin: %s\n", text);
+
+  return GNUNET_YES;
+}
+
 
 static void
 member_join_decision (void *cls,
@@ -135,11 +149,19 @@ member_join_decision (void *cls,
                       const struct GNUNET_PeerIdentity *relays,
                       const struct GNUNET_MessageHeader *join_msg)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+  struct GNUNET_MULTICAST_MemberTransmitHandle *req;
+  
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
               "Member received a decision from origin: %s\n", (GNUNET_YES == is_admitted)?"accepted":"rejected");
-
-  result = GNUNET_OK;
-  GNUNET_SCHEDULER_shutdown ();
+  
+  if (GNUNET_YES == is_admitted)
+  {
+    req = GNUNET_MULTICAST_member_to_origin (member,
+                                             0,
+                                             notify,
+                                             NULL);
+    
+  }
 }
 
 static void
@@ -161,6 +183,10 @@ member_message ()
 {
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "member message...\n");
+
+  // FIXME: not finished here
+  result = GNUNET_YES;
+  GNUNET_SCHEDULER_shutdown ();
 }
 
 static void
@@ -173,20 +199,21 @@ origin_join_request (void *cls,
 
   uint8_t data_size = ntohs (join_msg->size);
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Dizzy: Mh, got a join request...\n");
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "'%s'\n", (char *)&join_msg[1]);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Dizzy: Oh, it's Bird! Let's get him in.\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+              "origin got a join request...\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+              "origin receives: '%s'\n", (char *)&join_msg[1]);
 
-  char data[] = "Hi, Bird. Come in!";
+  char data[] = "Come in!";
   data_size = strlen (data) + 1;
   join_resp = GNUNET_malloc (sizeof (join_resp) + data_size);
   join_resp->size = htons (sizeof (join_resp) + data_size);
   join_resp->type = htons (123);
   GNUNET_memcpy (&join_resp[1], data, data_size);
 
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
+              "origin sends: '%s'\n", data);
+
   GNUNET_MULTICAST_join_decision (jh,
                                   GNUNET_YES,
                                   0,
@@ -198,37 +225,62 @@ origin_join_request (void *cls,
 
 static void
 origin_replay_frag (void *cls,
-                const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
-                uint64_t fragment_id,
-                uint64_t flags,
-                struct GNUNET_MULTICAST_ReplayHandle *rh)
+                    const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
+                    uint64_t fragment_id,
+                    uint64_t flags,
+                    struct GNUNET_MULTICAST_ReplayHandle *rh)
 {
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin replay fraq msg\n");
 }
 
 static void
 origin_replay_msg (void *cls,
-               const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
-               uint64_t message_id,
-               uint64_t fragment_offset,
-               uint64_t flags,
-               struct GNUNET_MULTICAST_ReplayHandle *rh)
+                   const struct GNUNET_CRYPTO_EcdsaPublicKey *member_pub_key,
+                   uint64_t message_id,
+                   uint64_t fragment_offset,
+                   uint64_t flags,
+                   struct GNUNET_MULTICAST_ReplayHandle *rh) 
 {
 
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin replay msg\n");
 }
 
+
+int
+origin_notify (void *cls, 
+               size_t *data_size, 
+               void *data)
+{
+  char text[] = "pong";
+  *data_size = strlen(text)+1;
+  memcpy(data, text, *data_size); 
+
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin sends (to all): %s\n", text);
+
+  return GNUNET_YES; 
+}
+
+
 static void
 origin_request (void *cls,
-            const struct GNUNET_MULTICAST_RequestHeader *req)
+                const struct GNUNET_MULTICAST_RequestHeader *req)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin request msg\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin receives: %s\n", (char *)&req[1]);
+  
+  if (0 != strncmp ("ping", (char *)&req[1], 4)) {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "origin didn't reveice a correct request");
+  }
 
+  GNUNET_MULTICAST_origin_to_all (origin,
+                                  0,
+                                  0,
+                                  origin_notify,
+                                  NULL);
 }
 
 static void
 origin_message (void *cls,
-            const struct GNUNET_MULTICAST_MessageHeader *msg)
+                const struct GNUNET_MULTICAST_MessageHeader *msg) 
 {
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, "origin message msg\n");
 }
@@ -268,8 +320,8 @@ multicast_ca1 (void *cls,
   // Get members keys
   member_key = GNUNET_CRYPTO_ecdsa_key_create ();
   GNUNET_CRYPTO_ecdsa_key_get_public (member_key, &member_pub_key);
-
-  char data[] = "Whut's up, Dizzy!";
+  
+  char data[] = "Hi, can I enter?";
   uint8_t data_size = strlen (data) + 1;
   join_msg = GNUNET_malloc (sizeof (join_msg) + data_size);
   join_msg->size = htons (sizeof (join_msg) + data_size);
@@ -453,7 +505,7 @@ testbed_master (void *cls,
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); /* Schedule a new task on shutdown */
 
   /* Schedule the shutdown task with a delay of a few Seconds */
-  timeout_tid = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 40),
+  timeout_tid = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 48),
                                               &timeout_task, NULL);
 }
 
diff --git a/src/namecache/Makefile.am b/src/namecache/Makefile.am
index 89d7bffee..d379b2602 100644
--- a/src/namecache/Makefile.am
+++ b/src/namecache/Makefile.am
@@ -121,6 +121,7 @@ libgnunet_plugin_namecache_sqlite_la_SOURCES = \
   plugin_namecache_sqlite.c
 libgnunet_plugin_namecache_sqlite_la_LIBADD = \
   libgnunetnamecache.la  \
+  $(top_builddir)/src/sq/libgnunetsq.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
   $(LTLIBINTL)
@@ -172,4 +173,3 @@ EXTRA_DIST = \
   test_plugin_namecache_sqlite.conf \
   test_plugin_namecache_postgres.conf \
         test_plugin_namecache_flat.conf
-
diff --git a/src/namecache/gnunet-namecache.c b/src/namecache/gnunet-namecache.c
index 490197b1e..2410c9b3c 100644
--- a/src/namecache/gnunet-namecache.c
+++ b/src/namecache/gnunet-namecache.c
@@ -225,13 +225,19 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'n', "name", "NAME",
-     gettext_noop ("name of the record to add/delete/display"), 1,
-     &GNUNET_GETOPT_set_string, &name},
-    {'z', "zone", "PKEY",
-     gettext_noop ("spezifies the public key of the zone to look in"), 1,
-     &GNUNET_GETOPT_set_string, &pkey},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_STRING ('n',
+                                 "name",
+                                 "NAME",
+                                 gettext_noop ("name of the record to add/delete/display"),
+                                 &name),
+
+    GNUNET_GETOPT_OPTION_STRING ('z',
+                                 "zone",
+                                 "PKEY",
+                                 gettext_noop ("spezifies the public key of the zone to look in"),
+                                 &pkey),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/namecache/plugin_namecache_sqlite.c b/src/namecache/plugin_namecache_sqlite.c
index fdce899fa..2f7b2a981 100644
--- a/src/namecache/plugin_namecache_sqlite.c
+++ b/src/namecache/plugin_namecache_sqlite.c
@@ -25,6 +25,7 @@
  */
 
 #include "platform.h"
+#include "gnunet_sq_lib.h"
 #include "gnunet_namecache_plugin.h"
 #include "gnunet_namecache_service.h"
 #include "gnunet_gnsrecord_lib.h"
@@ -241,25 +242,28 @@ database_setup (struct Plugin *plugin)
   sqlite3_finalize (stmt);
   create_indices (plugin->dbh);
 
-  if ((sq_prepare
-       (plugin->dbh,
-        "INSERT INTO ns096blocks (query,block,expiration_time) VALUES (?, ?, ?)",
-        &plugin->cache_block) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "DELETE FROM ns096blocks WHERE expiration_time<?",
-        &plugin->expire_blocks) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "DELETE FROM ns096blocks WHERE query=? AND expiration_time<=?",
-        &plugin->delete_block) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT block FROM ns096blocks WHERE query=? ORDER BY expiration_time DESC LIMIT 1",
-        &plugin->lookup_block) != SQLITE_OK)
+  if ( (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "INSERT INTO ns096blocks (query,block,expiration_time) VALUES (?, ?, ?)",
+                    &plugin->cache_block)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "DELETE FROM ns096blocks WHERE expiration_time<?",
+                    &plugin->expire_blocks)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "DELETE FROM ns096blocks WHERE query=? AND expiration_time<=?",
+                    &plugin->delete_block)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT block FROM ns096blocks WHERE query=? "
+                    "ORDER BY expiration_time DESC LIMIT 1",
+                    &plugin->lookup_block) )
       )
   {
-    LOG_SQLITE (plugin,GNUNET_ERROR_TYPE_ERROR, "precompiling");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "precompiling");
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -319,35 +323,41 @@ static void
 namecache_sqlite_expire_blocks (struct Plugin *plugin)
 {
   struct GNUNET_TIME_Absolute now;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_end
+  };
   int n;
 
   now = GNUNET_TIME_absolute_get ();
-  if (SQLITE_OK != sqlite3_bind_int64 (plugin->expire_blocks,
-                                       1, now.abs_value_us))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->expire_blocks,
+                      params))
   {
     LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->expire_blocks))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->expire_blocks);
     return;
   }
   n = sqlite3_step (plugin->expire_blocks);
-  if (SQLITE_OK != sqlite3_reset (plugin->expire_blocks))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->expire_blocks);
   switch (n)
   {
   case SQLITE_DONE:
-    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Records expired\n");
+    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                     "sqlite",
+                     "Records expired\n");
     return;
   case SQLITE_BUSY:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return;
   default:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return;
   }
@@ -368,8 +378,21 @@ namecache_sqlite_cache_block (void *cls,
   struct Plugin *plugin = cls;
   struct GNUNET_HashCode query;
   struct GNUNET_TIME_Absolute expiration;
-  int64_t dval;
-  size_t block_size;
+  size_t block_size = ntohl (block->purpose.size) +
+    sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) +
+    sizeof (struct GNUNET_CRYPTO_EcdsaSignature);
+  struct GNUNET_SQ_QueryParam del_params[] = {
+    GNUNET_SQ_query_param_auto_from_type (&query),
+    GNUNET_SQ_query_param_absolute_time (&expiration),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_QueryParam ins_params[] = {
+    GNUNET_SQ_query_param_auto_from_type (&query),
+    GNUNET_SQ_query_param_fixed_size (block,
+                                      block_size),
+    GNUNET_SQ_query_param_absolute_time (&expiration),
+    GNUNET_SQ_query_param_end
+  };
   int n;
 
   namecache_sqlite_expire_blocks (plugin);
@@ -377,12 +400,6 @@ namecache_sqlite_cache_block (void *cls,
                       sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
                       &query);
   expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time);
-  dval = (int64_t) expiration.abs_value_us;
-  if (dval < 0)
-    dval = INT64_MAX;
-  block_size = ntohl (block->purpose.size) +
-    sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) +
-    sizeof (struct GNUNET_CRYPTO_EcdsaSignature);
   if (block_size > 64 * 65536)
   {
     GNUNET_break (0);
@@ -390,61 +407,48 @@ namecache_sqlite_cache_block (void *cls,
   }
 
   /* delete old version of the block */
-  if ( (SQLITE_OK !=
-        sqlite3_bind_blob (plugin->delete_block, 1,
-                           &query, sizeof (struct GNUNET_HashCode),
-                           SQLITE_STATIC)) ||
-       (SQLITE_OK !=
-        sqlite3_bind_int64 (plugin->delete_block,
-                            2, dval)) )
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->delete_block,
+                      del_params))
   {
     LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->delete_block))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->delete_block);
     return GNUNET_SYSERR;
   }
   n = sqlite3_step (plugin->delete_block);
   switch (n)
   {
   case SQLITE_DONE:
-    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Old block deleted\n");
+    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                     "sqlite",
+                     "Old block deleted\n");
     break;
   case SQLITE_BUSY:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     break;
   default:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     break;
   }
-  if (SQLITE_OK != sqlite3_reset (plugin->delete_block))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->delete_block);
 
   /* insert new version of the block */
-  if ((SQLITE_OK !=
-       sqlite3_bind_blob (plugin->cache_block, 1,
-                          &query, sizeof (struct GNUNET_HashCode),
-                          SQLITE_STATIC)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_blob (plugin->cache_block, 2,
-                          block, block_size,
-                          SQLITE_STATIC)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_int64 (plugin->cache_block, 3,
-                           dval)))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->cache_block,
+                      ins_params))
   {
     LOG_SQLITE (plugin,
                 GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->cache_block))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->cache_block);
     return GNUNET_SYSERR;
 
   }
@@ -452,9 +456,8 @@ namecache_sqlite_cache_block (void *cls,
               "Caching block under derived key `%s'\n",
               GNUNET_h2s_full (&query));
   n = sqlite3_step (plugin->cache_block);
-  if (SQLITE_OK != sqlite3_reset (plugin->cache_block))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->cache_block);
   switch (n)
   {
   case SQLITE_DONE:
@@ -462,11 +465,13 @@ namecache_sqlite_cache_block (void *cls,
          "Record stored\n");
     return GNUNET_OK;
   case SQLITE_BUSY:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return GNUNET_NO;
   default:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return GNUNET_SYSERR;
   }
@@ -486,45 +491,63 @@ namecache_sqlite_cache_block (void *cls,
 static int
 namecache_sqlite_lookup_block (void *cls,
                                const struct GNUNET_HashCode *query,
-                               GNUNET_NAMECACHE_BlockCallback iter, void *iter_cls)
+                               GNUNET_NAMECACHE_BlockCallback iter,
+                               void *iter_cls)
 {
   struct Plugin *plugin = cls;
   int ret;
   int sret;
   size_t block_size;
   const struct GNUNET_GNSRECORD_Block *block;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_auto_from_type (query),
+    GNUNET_SQ_query_param_end
+  };
+  struct GNUNET_SQ_ResultSpec rs[] = {
+    GNUNET_SQ_result_spec_variable_size ((void **) &block,
+                                         &block_size),
+    GNUNET_SQ_result_spec_end
+  };
 
-  if (SQLITE_OK != sqlite3_bind_blob (plugin->lookup_block, 1,
-                                      query, sizeof (struct GNUNET_HashCode),
-                                      SQLITE_STATIC))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->lookup_block,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (plugin->lookup_block))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->lookup_block);
     return GNUNET_SYSERR;
   }
   ret = GNUNET_NO;
-  if (SQLITE_ROW == (sret = sqlite3_step (plugin->lookup_block)))
+  if (SQLITE_ROW ==
+      (sret = sqlite3_step (plugin->lookup_block)))
   {
-    block = sqlite3_column_blob (plugin->lookup_block, 0);
-    block_size = sqlite3_column_bytes (plugin->lookup_block, 0);
-    if ( (block_size < sizeof (struct GNUNET_GNSRECORD_Block)) ||
-         (ntohl (block->purpose.size) +
-          sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) +
-          sizeof (struct GNUNET_CRYPTO_EcdsaSignature) != block_size) )
+    if (GNUNET_OK !=
+        GNUNET_SQ_extract_result (plugin->lookup_block,
+                                  rs))
     {
       GNUNET_break (0);
       ret = GNUNET_SYSERR;
     }
+    else if ( (block_size < sizeof (struct GNUNET_GNSRECORD_Block)) ||
+              (ntohl (block->purpose.size) +
+               sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) +
+               sizeof (struct GNUNET_CRYPTO_EcdsaSignature) != block_size) )
+    {
+      GNUNET_break (0);
+      GNUNET_SQ_cleanup_result (rs);
+      ret = GNUNET_SYSERR;
+    }
     else
     {
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Found block under derived key `%s'\n",
                   GNUNET_h2s_full (query));
-      iter (iter_cls, block);
+      iter (iter_cls,
+            block);
+      GNUNET_SQ_cleanup_result (rs);
       ret = GNUNET_YES;
     }
   }
@@ -532,7 +555,9 @@ namecache_sqlite_lookup_block (void *cls,
   {
     if (SQLITE_DONE != sret)
     {
-      LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
+      LOG_SQLITE (plugin,
+                  GNUNET_ERROR_TYPE_ERROR,
+                  "sqlite_step");
       ret = GNUNET_SYSERR;
     }
     else
@@ -542,10 +567,8 @@ namecache_sqlite_lookup_block (void *cls,
                   GNUNET_h2s_full (query));
     }
   }
-  if (SQLITE_OK != sqlite3_reset (plugin->lookup_block))
-    LOG_SQLITE (plugin,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   plugin->lookup_block);
   return ret;
 }
 
diff --git a/src/namestore/Makefile.am b/src/namestore/Makefile.am
index e9b0f179a..de46e9c89 100644
--- a/src/namestore/Makefile.am
+++ b/src/namestore/Makefile.am
@@ -189,6 +189,7 @@ libgnunet_plugin_namestore_sqlite_la_SOURCES = \
   plugin_namestore_sqlite.c
 libgnunet_plugin_namestore_sqlite_la_LIBADD = \
   $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la  \
+  $(top_builddir)/src/sq/libgnunetsq.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
   $(LTLIBINTL)
diff --git a/src/namestore/gnunet-namestore.c b/src/namestore/gnunet-namestore.c
index 457e77022..2ce3741f8 100644
--- a/src/namestore/gnunet-namestore.c
+++ b/src/namestore/gnunet-namestore.c
@@ -1130,49 +1130,88 @@ main (int argc,
   is_public = -1;
   is_shadow = -1;
 
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "add", NULL,
-     gettext_noop ("add record"), 0,
-     &GNUNET_GETOPT_set_one, &add},
-    {'d', "delete", NULL,
-     gettext_noop ("delete record"), 0,
-     &GNUNET_GETOPT_set_one, &del},
-    {'D', "display", NULL,
-     gettext_noop ("display records"), 0,
-     &GNUNET_GETOPT_set_one, &list},
-    {'e', "expiration", "TIME",
-     gettext_noop ("expiration time for record to use (for adding only), \"never\" is possible"), 1,
-     &GNUNET_GETOPT_set_string, &expirationstring},
-    {'i', "nick", "NICKNAME",
-     gettext_noop ("set the desired nick name for the zone"), 1,
-     &GNUNET_GETOPT_set_string, &nickstring},
-    {'m', "monitor", NULL,
-     gettext_noop ("monitor changes in the namestore"), 0,
-     &GNUNET_GETOPT_set_one, &monitor},
-    {'n', "name", "NAME",
-     gettext_noop ("name of the record to add/delete/display"), 1,
-     &GNUNET_GETOPT_set_string, &name},
-    {'r', "reverse", "PKEY",
-     gettext_noop ("determine our name for the given PKEY"), 1,
-     &GNUNET_GETOPT_set_string, &reverse_pkey},
-    {'t', "type", "TYPE",
-     gettext_noop ("type of the record to add/delete/display"), 1,
-     &GNUNET_GETOPT_set_string, &typestring},
-    {'u', "uri", "URI",
-     gettext_noop ("URI to import into our zone"), 1,
-     &GNUNET_GETOPT_set_string, &uri},
-    {'V', "value", "VALUE",
-     gettext_noop ("value of the record to add/delete"), 1,
-     &GNUNET_GETOPT_set_string, &value},
-    {'p', "public", NULL,
-     gettext_noop ("create or list public record"), 0,
-     &GNUNET_GETOPT_set_one, &is_public},
-    {'s', "shadow", NULL,
-     gettext_noop ("create shadow record (only valid if all other records of the same type have expired"), 0,
-     &GNUNET_GETOPT_set_one, &is_shadow},
-    {'z', "zone", "EGO",
-     gettext_noop ("name of the ego controlling the zone"), 1,
-     &GNUNET_GETOPT_set_string, &ego_name},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('a',
+                                  "add",
+                                  gettext_noop ("add record"),
+                                  &add),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('d',
+                                  "delete",
+                                  gettext_noop ("delete record"),
+                                  &del),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('D',
+                                  "display",
+                                  gettext_noop ("display records"),
+                                  &list), 
+
+    GNUNET_GETOPT_OPTION_STRING ('e',
+                                 "expiration",
+                                 "TIME",
+                                 gettext_noop ("expiration time for record to use (for adding only), \"never\" is possible"),
+                                 &expirationstring),
+
+    GNUNET_GETOPT_OPTION_STRING ('i',
+                                 "nick",
+                                 "NICKNAME",
+                                 gettext_noop ("set the desired nick name for the zone"),
+                                 &nickstring), 
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('m',
+                                  "monitor",
+                                  gettext_noop ("monitor changes in the namestore"),
+                                  &monitor),
+
+    GNUNET_GETOPT_OPTION_STRING ('n',
+                                 "name",
+                                 "NAME",
+                                 gettext_noop ("name of the record to add/delete/display"),
+                                 &name),
+
+    GNUNET_GETOPT_OPTION_STRING ('r',
+                                 "reverse",
+                                 "PKEY",
+                                 gettext_noop ("determine our name for the given PKEY"),
+                                 &reverse_pkey),
+
+
+
+    GNUNET_GETOPT_OPTION_STRING ('t',
+                                 "type",
+                                 "TYPE",
+                                 gettext_noop ("type of the record to add/delete/display"),
+                                 &typestring),
+
+    GNUNET_GETOPT_OPTION_STRING ('u',
+                                 "uri",
+                                 "URI",
+                                 gettext_noop ("URI to import into our zone"),
+                                 &uri),
+
+    GNUNET_GETOPT_OPTION_STRING ('V',
+                                 "value",
+                                 "VALUE",
+                                 gettext_noop ("value of the record to add/delete"),
+                                 &value),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('p',
+                                  "public",
+                                  gettext_noop ("create or list public record"),
+                                  &is_public),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('s',
+                                  "shadow",
+                                  gettext_noop ("create shadow record (only valid if all other records of the same type have expired"),
+                                  &is_shadow),
+
+    GNUNET_GETOPT_OPTION_STRING ('z',
+                                 "zone",
+                                 "EGO",
+                                 gettext_noop ("name of the ego controlling the zone"),
+                                 &ego_name), 
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index ae65802b0..6cb4290a0 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -1136,7 +1136,7 @@ handle_zone_to_name_it (void *cls,
   name_len = (NULL == name) ? 0 : strlen (name) + 1;
   rd_ser_len = GNUNET_GNSRECORD_records_get_size (rd_count, rd);
   msg_size = sizeof (struct ZoneToNameResponseMessage) + name_len + rd_ser_len;
-  if (msg_size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (msg_size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     ztn_ctx->success = GNUNET_SYSERR;
diff --git a/src/namestore/plugin_namestore_sqlite.c b/src/namestore/plugin_namestore_sqlite.c
index d64ce10a8..5c3533506 100644
--- a/src/namestore/plugin_namestore_sqlite.c
+++ b/src/namestore/plugin_namestore_sqlite.c
@@ -1,6 +1,6 @@
  /*
   * This file is part of GNUnet
-  * Copyright (C) 2009-2013 GNUnet e.V.
+  * Copyright (C) 2009-2017 GNUnet e.V.
   *
   * GNUnet is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License as published
@@ -28,6 +28,7 @@
 #include "gnunet_namestore_plugin.h"
 #include "gnunet_namestore_service.h"
 #include "gnunet_gnsrecord_lib.h"
+#include "gnunet_sq_lib.h"
 #include "namestore.h"
 #include <sqlite3.h>
 
@@ -113,16 +114,24 @@ struct Plugin
  * @return 0 on success
  */
 static int
-sq_prepare (sqlite3 * dbh, const char *zSql, sqlite3_stmt ** ppStmt)
+sq_prepare (sqlite3 *dbh,
+            const char *zSql,
+            sqlite3_stmt **ppStmt)
 {
   char *dummy;
   int result;
 
   result =
-      sqlite3_prepare_v2 (dbh, zSql, strlen (zSql), ppStmt,
+      sqlite3_prepare_v2 (dbh,
+                          zSql,
+                          strlen (zSql),
+                          ppStmt,
                           (const char **) &dummy);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Prepared `%s' / %p: %d\n", zSql, *ppStmt, result);
+       "Prepared `%s' / %p: %d\n",
+       zSql,
+       *ppStmt,
+       result);
   return result;
 }
 
@@ -275,38 +284,41 @@ database_setup (struct Plugin *plugin)
 
   create_indices (plugin->dbh);
 
-  if ((sq_prepare
-       (plugin->dbh,
-        "INSERT INTO ns097records (zone_private_key, pkey, rvalue, record_count, record_data, label)"
-        " VALUES (?, ?, ?, ?, ?, ?)",
-        &plugin->store_records) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "DELETE FROM ns097records WHERE zone_private_key=? AND label=?",
-        &plugin->delete_records) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT record_count,record_data,label"
-        " FROM ns097records WHERE zone_private_key=? AND pkey=?",
-        &plugin->zone_to_name) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT record_count,record_data,label"
-        " FROM ns097records WHERE zone_private_key=? ORDER BY rvalue LIMIT 1 OFFSET ?",
-        &plugin->iterate_zone) != SQLITE_OK) ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT record_count,record_data,label,zone_private_key"
-        " FROM ns097records ORDER BY rvalue LIMIT 1 OFFSET ?",
-        &plugin->iterate_all_zones) != SQLITE_OK)  ||
-      (sq_prepare
-       (plugin->dbh,
-        "SELECT record_count,record_data,label,zone_private_key"
-        " FROM ns097records WHERE zone_private_key=? AND label=?",
-        &plugin->lookup_label) != SQLITE_OK)
-      )
+  if ( (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "INSERT INTO ns097records (zone_private_key, pkey, rvalue, record_count, record_data, label)"
+                    " VALUES (?, ?, ?, ?, ?, ?)",
+                    &plugin->store_records)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "DELETE FROM ns097records WHERE zone_private_key=? AND label=?",
+                    &plugin->delete_records)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT record_count,record_data,label"
+                    " FROM ns097records WHERE zone_private_key=? AND pkey=?",
+                    &plugin->zone_to_name)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT record_count,record_data,label"
+                    " FROM ns097records WHERE zone_private_key=?"
+                    " ORDER BY rvalue LIMIT 1 OFFSET ?",
+                    &plugin->iterate_zone)) ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT record_count,record_data,label,zone_private_key"
+                    " FROM ns097records ORDER BY rvalue LIMIT 1 OFFSET ?",
+                    &plugin->iterate_all_zones))  ||
+       (SQLITE_OK !=
+        sq_prepare (plugin->dbh,
+                    "SELECT record_count,record_data,label,zone_private_key"
+                    " FROM ns097records WHERE zone_private_key=? AND label=?",
+                    &plugin->lookup_label))
+       )
   {
-    LOG_SQLITE (plugin,GNUNET_ERROR_TYPE_ERROR, "precompiling");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "precompiling");
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -341,21 +353,30 @@ database_shutdown (struct Plugin *plugin)
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
          _("Tried to close sqlite without finalizing all prepared statements.\n"));
-    stmt = sqlite3_next_stmt (plugin->dbh, NULL);
-    while (stmt != NULL)
+    stmt = sqlite3_next_stmt (plugin->dbh,
+                              NULL);
+    while (NULL != stmt)
     {
-      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite",
-                       "Closing statement %p\n", stmt);
+      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                       "sqlite",
+                       "Closing statement %p\n",
+                       stmt);
       result = sqlite3_finalize (stmt);
       if (result != SQLITE_OK)
-        GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite",
-                         "Failed to close statement %p: %d\n", stmt, result);
-      stmt = sqlite3_next_stmt (plugin->dbh, NULL);
+        GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                         "sqlite",
+                         "Failed to close statement %p: %d\n",
+                         stmt,
+                         result);
+      stmt = sqlite3_next_stmt (plugin->dbh,
+                                NULL);
     }
     result = sqlite3_close (plugin->dbh);
   }
   if (SQLITE_OK != result)
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite3_close");
 
   GNUNET_free_non_null (plugin->fn);
 }
@@ -407,7 +428,13 @@ namestore_sqlite_store_records (void *cls,
     return GNUNET_SYSERR;
   }
   {
+    /* First delete 'old' records */
     char data[data_size];
+    struct GNUNET_SQ_QueryParam dparams[] = {
+      GNUNET_SQ_query_param_auto_from_type (zone_key),
+      GNUNET_SQ_query_param_string (label),
+      GNUNET_SQ_query_param_end
+    };
 
     if (data_size !=
         GNUNET_GNSRECORD_records_serialize (rd_count,
@@ -418,92 +445,71 @@ namestore_sqlite_store_records (void *cls,
       GNUNET_break (0);
       return GNUNET_SYSERR;
     }
-
-    /* First delete 'old' records */
-    if ((SQLITE_OK !=
-         sqlite3_bind_blob (plugin->delete_records,
-                            1,
-                            zone_key,
-                            sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey),
-                            SQLITE_STATIC)) ||
-        (SQLITE_OK !=
-         sqlite3_bind_text (plugin->delete_records,
-                            2,
-                            label,
-                            -1,
-                            SQLITE_STATIC)))
+    if (GNUNET_OK !=
+        GNUNET_SQ_bind (plugin->delete_records,
+                        dparams))
     {
       LOG_SQLITE (plugin,
                   GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                   "sqlite3_bind_XXXX");
-      if (SQLITE_OK != sqlite3_reset (plugin->delete_records))
-        LOG_SQLITE (plugin,
-                    GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_reset");
+      GNUNET_SQ_reset (plugin->dbh,
+                       plugin->delete_records);
       return GNUNET_SYSERR;
 
     }
     n = sqlite3_step (plugin->delete_records);
-    if (SQLITE_OK != sqlite3_reset (plugin->delete_records))
-      LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->delete_records);
 
     if (0 != rd_count)
     {
-      if ((SQLITE_OK !=
-           sqlite3_bind_blob (plugin->store_records,
-                              1,
-                              zone_key,
-                              sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey),
-                              SQLITE_STATIC)) ||
-          (SQLITE_OK !=
-           sqlite3_bind_blob (plugin->store_records,
-                              2,
-                              &pkey,
-                              sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
-                              SQLITE_STATIC)) ||
-          (SQLITE_OK !=
-           sqlite3_bind_int64 (plugin->store_records, 3, rvalue)) ||
-          (SQLITE_OK !=
-           sqlite3_bind_int (plugin->store_records, 4, rd_count)) ||
-          (SQLITE_OK !=
-           sqlite3_bind_blob (plugin->store_records, 5,
-                              data, data_size,
-                              SQLITE_STATIC)) ||
-          (SQLITE_OK !=
-           sqlite3_bind_text (plugin->store_records, 6,
-                              label, -1,
-                              SQLITE_STATIC)))
+      uint32_t rd_count32 = (uint32_t) rd_count;
+      struct GNUNET_SQ_QueryParam sparams[] = {
+        GNUNET_SQ_query_param_auto_from_type (zone_key),
+        GNUNET_SQ_query_param_auto_from_type (&rvalue),
+        GNUNET_SQ_query_param_uint64 (&rvalue),
+        GNUNET_SQ_query_param_uint32 (&rd_count32),
+        GNUNET_SQ_query_param_fixed_size (data, data_size),
+        GNUNET_SQ_query_param_string (label),
+        GNUNET_SQ_query_param_end
+      };
+
+      if (GNUNET_OK !=
+          GNUNET_SQ_bind (plugin->store_records,
+                          sparams))
       {
         LOG_SQLITE (plugin,
                     GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                     "sqlite3_bind_XXXX");
-        if (SQLITE_OK != sqlite3_reset (plugin->store_records))
-          LOG_SQLITE (plugin,
-                      GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                      "sqlite3_reset");
+        GNUNET_SQ_reset (plugin->dbh,
+                         plugin->store_records);
         return GNUNET_SYSERR;
       }
       n = sqlite3_step (plugin->store_records);
-      if (SQLITE_OK != sqlite3_reset (plugin->store_records))
-        LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                    "sqlite3_reset");
+      GNUNET_SQ_reset (plugin->dbh,
+                       plugin->store_records);
     }
   }
   switch (n)
   {
   case SQLITE_DONE:
     if (0 != rd_count)
-      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Record stored\n");
+      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                       "sqlite",
+                       "Record stored\n");
     else
-      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Record deleted\n");
+      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                       "sqlite",
+                       "Record deleted\n");
     return GNUNET_OK;
   case SQLITE_BUSY:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return GNUNET_NO;
   default:
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
     return GNUNET_SYSERR;
   }
@@ -526,37 +532,47 @@ static int
 get_record_and_call_iterator (struct Plugin *plugin,
                               sqlite3_stmt *stmt,
                               const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
-                              GNUNET_NAMESTORE_RecordIterator iter, void *iter_cls)
+                              GNUNET_NAMESTORE_RecordIterator iter,
+                              void *iter_cls)
 {
-  unsigned int record_count;
+  uint32_t record_count;
   size_t data_size;
-  const char *data;
-  const char *label;
+  void *data;
+  char *label;
+  struct GNUNET_CRYPTO_EcdsaPrivateKey zk;
   int ret;
   int sret;
 
   ret = GNUNET_NO;
   if (SQLITE_ROW == (sret = sqlite3_step (stmt)))
   {
-    record_count = sqlite3_column_int (stmt, 0);
-    data_size = sqlite3_column_bytes (stmt, 1);
-    data = sqlite3_column_blob (stmt, 1);
-    label = (const char*) sqlite3_column_text (stmt, 2);
+    struct GNUNET_SQ_ResultSpec rs[] = {
+      GNUNET_SQ_result_spec_uint32 (&record_count),
+      GNUNET_SQ_result_spec_variable_size (&data, &data_size),
+      GNUNET_SQ_result_spec_string (&label),
+      GNUNET_SQ_result_spec_end
+    };
+    struct GNUNET_SQ_ResultSpec rsx[] = {
+      GNUNET_SQ_result_spec_uint32 (&record_count),
+      GNUNET_SQ_result_spec_variable_size (&data, &data_size),
+      GNUNET_SQ_result_spec_string (&label),
+      GNUNET_SQ_result_spec_auto_from_type (&zk),
+      GNUNET_SQ_result_spec_end
+    };
+
     if (NULL == zone_key)
     {
-      /* must be "iterate_all_zones", got one extra return value */
-      if (sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey) !=
-          sqlite3_column_bytes (stmt, 3))
-      {
-        GNUNET_break (0);
-        ret = GNUNET_SYSERR;
-      }
-      else
-      {
-        zone_key = sqlite3_column_blob (stmt, 3);
-      }
+      zone_key = &zk;
+      ret = GNUNET_SQ_extract_result (stmt,
+                                      rsx);
+    }
+    else
+    {
+      ret = GNUNET_SQ_extract_result (stmt,
+                                      rs);
     }
-    if (record_count > 64 * 1024)
+    if ( (GNUNET_OK != ret) ||
+         (record_count > 64 * 1024) )
     {
       /* sanity check, don't stack allocate far too much just
          because database might contain a large value here */
@@ -568,32 +584,40 @@ get_record_and_call_iterator (struct Plugin *plugin,
       struct GNUNET_GNSRECORD_Data rd[record_count];
 
       if (GNUNET_OK !=
-          GNUNET_GNSRECORD_records_deserialize (data_size, data,
-                                                record_count, rd))
+          GNUNET_GNSRECORD_records_deserialize (data_size,
+                                                data,
+                                                record_count,
+                                                rd))
       {
         GNUNET_break (0);
         ret = GNUNET_SYSERR;
       }
-      else if (NULL != zone_key)
+      else
       {
         if (NULL != iter)
-                iter (iter_cls, zone_key, label, record_count, rd);
+          iter (iter_cls,
+                zone_key,
+                label,
+                record_count,
+                rd);
         ret = GNUNET_YES;
       }
     }
+    GNUNET_SQ_cleanup_result (rs);
   }
   else
   {
     if (SQLITE_DONE != sret)
-      LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
+      LOG_SQLITE (plugin,
+                  GNUNET_ERROR_TYPE_ERROR,
+                  "sqlite_step");
   }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-    LOG_SQLITE (plugin,
-                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   return ret;
 }
 
+
 /**
  * Lookup records in the datastore for which we are the authority.
  *
@@ -606,37 +630,35 @@ get_record_and_call_iterator (struct Plugin *plugin,
  */
 static int
 namestore_sqlite_lookup_records (void *cls,
-    const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, const char *label,
-    GNUNET_NAMESTORE_RecordIterator iter, void *iter_cls)
+                                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
+                                 const char *label,
+                                 GNUNET_NAMESTORE_RecordIterator iter,
+                                 void *iter_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
-  int err;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_auto_from_type (zone),
+    GNUNET_SQ_query_param_string (label),
+    GNUNET_SQ_query_param_end
+  };
 
   if (NULL == zone)
-  {
     return GNUNET_SYSERR;
-  }
-  else
-  {
-    stmt = plugin->lookup_label;
-    err = ( (SQLITE_OK != sqlite3_bind_blob (stmt, 1,
-                                             zone, sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey),
-                                             SQLITE_STATIC)) ||
-            (SQLITE_OK != sqlite3_bind_text (stmt, 2,
-                                              label, -1, SQLITE_STATIC)) );
-  }
-  if (err)
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->lookup_label,
+                      params))
   {
     LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->lookup_label);
     return GNUNET_SYSERR;
   }
-  return get_record_and_call_iterator (plugin, stmt, zone, iter, iter_cls);
+  return get_record_and_call_iterator (plugin,
+                                       plugin->lookup_label,
+                                       zone,
+                                       iter,
+                                       iter_cls);
 }
 
 
@@ -655,7 +677,8 @@ static int
 namestore_sqlite_iterate_records (void *cls,
                                   const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
                                   uint64_t offset,
-                                  GNUNET_NAMESTORE_RecordIterator iter, void *iter_cls)
+                                  GNUNET_NAMESTORE_RecordIterator iter,
+                                  void *iter_cls)
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt;
@@ -663,30 +686,41 @@ namestore_sqlite_iterate_records (void *cls,
 
   if (NULL == zone)
   {
+    struct GNUNET_SQ_QueryParam params[] = {
+      GNUNET_SQ_query_param_uint64 (&offset),
+      GNUNET_SQ_query_param_end
+    };
+
     stmt = plugin->iterate_all_zones;
-    err = (SQLITE_OK != sqlite3_bind_int64 (stmt, 1,
-                                            offset));
+    err = GNUNET_SQ_bind (stmt,
+                          params);
   }
   else
   {
+    struct GNUNET_SQ_QueryParam params[] = {
+      GNUNET_SQ_query_param_auto_from_type (zone),
+      GNUNET_SQ_query_param_uint64 (&offset),
+      GNUNET_SQ_query_param_end
+    };
+
     stmt = plugin->iterate_zone;
-    err = ( (SQLITE_OK != sqlite3_bind_blob (stmt, 1,
-                                             zone, sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey),
-                                             SQLITE_STATIC)) ||
-            (SQLITE_OK != sqlite3_bind_int64 (stmt, 2,
-                                              offset)) );
+    err = GNUNET_SQ_bind (stmt,
+                          params);
   }
-  if (err)
+  if (GNUNET_OK != err)
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
     return GNUNET_SYSERR;
   }
-  return get_record_and_call_iterator (plugin, stmt, zone, iter, iter_cls);
+  return get_record_and_call_iterator (plugin,
+                                       stmt,
+                                       zone,
+                                       iter,
+                                       iter_cls);
 }
 
 
@@ -708,29 +742,31 @@ namestore_sqlite_zone_to_name (void *cls,
                                GNUNET_NAMESTORE_RecordIterator iter, void *iter_cls)
 {
   struct Plugin *plugin = cls;
-  sqlite3_stmt *stmt;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_auto_from_type (zone),
+    GNUNET_SQ_query_param_auto_from_type (value_zone),
+    GNUNET_SQ_query_param_end
+  };
 
-  stmt = plugin->zone_to_name;
-  if ( (SQLITE_OK != sqlite3_bind_blob (stmt, 1,
-                                        zone, sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey),
-                                        SQLITE_STATIC)) ||
-       (SQLITE_OK != sqlite3_bind_blob (stmt, 2,
-                                        value_zone, sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
-                                        SQLITE_STATIC)) )
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (plugin->zone_to_name,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin,
-                  GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     plugin->zone_to_name);
     return GNUNET_SYSERR;
   }
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Performing reverse lookup for `%s'\n",
        GNUNET_GNSRECORD_z2s (value_zone));
-
-  return get_record_and_call_iterator (plugin, stmt, zone, iter, iter_cls);
+  return get_record_and_call_iterator (plugin,
+                                       plugin->zone_to_name,
+                                       zone,
+                                       iter,
+                                       iter_cls);
 }
 
 
diff --git a/src/nat-auto/Makefile.am b/src/nat-auto/Makefile.am
index 19695fabd..14f3f2330 100644
--- a/src/nat-auto/Makefile.am
+++ b/src/nat-auto/Makefile.am
@@ -37,11 +37,11 @@ lib_LTLIBRARIES = \
 
 libgnunetnatauto_la_SOURCES = \
   nat_auto_api.c \
-  nat_auto_api_test.c 
+  nat_auto_api_test.c
 libgnunetnatauto_la_LIBADD = \
   $(top_builddir)/src/nat/libgnunetnatnew.la \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(GN_LIBINTL) @EXT_LIBS@ 
+  $(GN_LIBINTL) @EXT_LIBS@
 libgnunetnatauto_la_LDFLAGS = \
   $(GN_LIB_LDFLAGS) $(WINFLAGS) \
   -version-info 0:0:0
@@ -55,4 +55,3 @@ gnunet_service_nat_auto_LDADD = \
   $(LIBGCRYPT_LIBS) \
   -lgcrypt \
   $(GN_LIBINTL)
-
diff --git a/src/nat-auto/gnunet-nat-auto.c b/src/nat-auto/gnunet-nat-auto.c
index 9ba81eb5f..8b281d1ab 100644
--- a/src/nat-auto/gnunet-nat-auto.c
+++ b/src/nat-auto/gnunet-nat-auto.c
@@ -63,7 +63,7 @@ static char *section_name;
 /**
  * Should we run autoconfiguration?
  */
-static unsigned int do_auto;
+static int do_auto;
 
 /**
  * Handle to a NAT test operation.
@@ -174,6 +174,9 @@ auto_config_cb (void *cls,
               GNUNET_NAT_AUTO_status2string (result),
               nat_type);
 
+  if (NULL == diff)
+    return;
+
   /* Shortcut: if there are no changes suggested, bail out early. */
   if (GNUNET_NO ==
       GNUNET_CONFIGURATION_is_dirty (diff))
@@ -186,20 +189,16 @@ auto_config_cb (void *cls,
      to the user */
   new_cfg = write_cfg ? GNUNET_CONFIGURATION_dup (cfg) : NULL;
 
-  if (NULL != diff)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
-                _("Suggested configuration changes:\n"));
-    GNUNET_CONFIGURATION_iterate_section_values (diff,
-                                                 "nat",
-                                                 &auto_conf_iter,
-                                                 new_cfg);
-  }
+  GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
+              _("Suggested configuration changes:\n"));
+  GNUNET_CONFIGURATION_iterate_section_values (diff,
+                                               "nat",
+                                               &auto_conf_iter,
+                                               new_cfg);
 
   /* If desired, write configuration to file; we write only the
      changes to the defaults to keep things compact. */
-  if ( (write_cfg) &&
-       (NULL != diff) )
+  if (write_cfg)
   {
     struct GNUNET_CONFIGURATION_Handle *def_cfg;
 
@@ -298,8 +297,8 @@ run (void *cls,
   if (do_auto)
   {
     ah = GNUNET_NAT_AUTO_autoconfig_start (c,
-                                      &auto_config_cb,
-                                      NULL);
+                                           &auto_config_cb,
+                                           NULL);
   }
 
   if (use_tcp && use_udp)
@@ -340,22 +339,32 @@ int
 main (int argc,
       char *const argv[])
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'a', "auto", NULL,
-     gettext_noop ("run autoconfiguration"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &do_auto },
-    {'S', "section", "NAME",
-     gettext_noop ("section name providing the configuration for the adapter"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &section_name },
-    {'t', "tcp", NULL,
-     gettext_noop ("use TCP"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &use_tcp },
-    {'u', "udp", NULL,
-     gettext_noop ("use UDP"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &use_udp },
-    {'w', "write", NULL,
-     gettext_noop ("write configuration file (for autoconfiguration)"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &write_cfg },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('a',
+                                  "auto",
+                                  gettext_noop ("run autoconfiguration"),
+                                  &do_auto),
+
+    GNUNET_GETOPT_OPTION_STRING ('S',
+                                 "section",
+                                 "NAME",
+                                 gettext_noop ("section name providing the configuration for the adapter"),
+                                 &section_name),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('t',
+                                   "tcp",
+                                   gettext_noop ("use TCP"),
+                                   &use_tcp),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('u',
+                                   "udp",
+                                   gettext_noop ("use UDP"),
+                                   &use_udp),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('w',
+                                   "write",
+                                   gettext_noop ("write configuration file (for autoconfiguration)"),
+                                   &write_cfg),
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/nat-auto/gnunet-nat-server.c b/src/nat-auto/gnunet-nat-server.c
index 371e4b27e..590fad4d6 100644
--- a/src/nat-auto/gnunet-nat-server.c
+++ b/src/nat-auto/gnunet-nat-server.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
+     Copyright (C) 2011, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -31,9 +31,21 @@
 
 
 /**
- * Our server.
+ * Information we track per client.
  */
-static struct GNUNET_SERVER_Handle *server;
+struct ClientData
+{
+  /**
+   * Timeout task.
+   */
+  struct GNUNET_SCHEDULER_Task *tt;
+
+  /**
+   * Client handle.
+   */
+  struct GNUNET_SERVICE_Client *client;
+};
+
 
 /**
  * Our configuration.
@@ -248,21 +260,18 @@ try_send_udp (uint32_t dst_ipv4,
  * We've received a request to probe a NAT
  * traversal. Do it.
  *
- * @param cls unused
- * @param client handle to client (we always close)
+ * @param cls handle to client (we always close)
  * @param msg message with details about what to test
  */
 static void
-test (void *cls,
-      struct GNUNET_SERVER_Client *client,
-      const struct GNUNET_MessageHeader *msg)
+handle_test (void *cls,
+             const struct GNUNET_NAT_AUTO_TestMessage *tm)
 {
-  const struct GNUNET_NAT_AUTO_TestMessage *tm;
+  struct ClientData *cd = cls;
   uint16_t dport;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received test request\n");
-  tm = (const struct GNUNET_NAT_AUTO_TestMessage *) msg;
   dport = ntohs (tm->dport);
   if (0 == dport)
     try_anat (tm->dst_ipv4,
@@ -276,126 +285,119 @@ test (void *cls,
     try_send_udp (tm->dst_ipv4,
                   dport,
                   tm->data);
-  GNUNET_SERVER_receive_done (client,
-                              GNUNET_NO);
+  GNUNET_SERVICE_client_drop (cd->client);
 }
 
 
 /**
- * Task run during shutdown.
+ * Main function that will be run.
  *
- * @param cls unused
+ * @param cls closure
+ * @param c configuration
+ * @param srv service handle
  */
 static void
-shutdown_task (void *cls)
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_SERVICE_Handle *srv)
 {
-  GNUNET_SERVER_destroy (server);
-  server = NULL;
+  cfg = c;
 }
 
 
 /**
- * Main function that will be run.
+ * Forcefully drops client after 1s.
  *
- * @param cls closure
- * @param args remaining command-line arguments
- * @param cfgfile name of the configuration file used (for saving, can be NULL!)
- * @param c configuration
+ * @param cls our `struct ClientData` of a client to drop
  */
 static void
-run (void *cls,
-     char *const *args,
-     const char *cfgfile,
-     const struct GNUNET_CONFIGURATION_Handle *c)
+force_timeout (void *cls)
 {
-  static const struct GNUNET_SERVER_MessageHandler handlers[] = {
-    {&test, NULL, GNUNET_MESSAGE_TYPE_NAT_TEST,
-     sizeof (struct GNUNET_NAT_AUTO_TestMessage)},
-    {NULL, NULL, 0, 0}
-  };
-  unsigned int port;
-  struct sockaddr_in in4;
-  struct sockaddr_in6 in6;
-
-  socklen_t slen[] = {
-    sizeof (in4),
-    sizeof (in6),
-    0
-  };
-  struct sockaddr *sa[] = {
-    (struct sockaddr *) &in4,
-    (struct sockaddr *) &in6,
-    NULL
-  };
+  struct ClientData *cd = cls;
 
-  cfg = c;
-  if ( (NULL == args[0]) ||
-       (1 != SSCANF (args[0], "%u", &port)) ||
-       (0 == port) ||
-       (65536 <= port) )
-  {
-    FPRINTF (stderr,
-             _("Please pass valid port number as the first argument! (got `%s')\n"),
-             args[0]);
-    return;
-  }
-  memset (&in4, 0, sizeof (in4));
-  memset (&in6, 0, sizeof (in6));
-  in4.sin_family = AF_INET;
-  in4.sin_port = htons ((uint16_t) port);
-  in6.sin6_family = AF_INET6;
-  in6.sin6_port = htons ((uint16_t) port);
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  in4.sin_len = sizeof (in4);
-  in6.sin6_len = sizeof (in6);
-#endif
-  server = GNUNET_SERVER_create (NULL,
-                                 NULL,
-                                 (struct sockaddr * const *) sa,
-                                 slen,
-                                 GNUNET_TIME_UNIT_SECONDS,
-                                 GNUNET_YES);
-  GNUNET_SERVER_add_handlers (server,
-                              handlers);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
-                                 NULL);
+  cd->tt = NULL;
+  GNUNET_SERVICE_client_drop (cd->client);
 }
 
 
+
 /**
- * Main function of gnunet-nat-server.
+ * Callback called when a client connects to the service.
  *
- * @param argc number of command-line arguments
- * @param argv command line
- * @return 0 on success, -1 on error
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return our `struct ClientData`
  */
-int
-main (int argc, char *const argv[])
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    GNUNET_GETOPT_OPTION_END
-  };
-
-  if (GNUNET_OK !=
-      GNUNET_STRINGS_get_utf8_args (argc, argv,
-                                    &argc, &argv))
-    return 2;
-
-  if (GNUNET_OK !=
-      GNUNET_PROGRAM_run (argc,
-                          argv,
-                          "gnunet-nat-server [options] PORT",
-                          _("GNUnet NAT traversal test helper daemon"),
-                          options,
-                          &run,
-                          NULL))
-  {
-    GNUNET_free ((void*) argv);
-    return 1;
-  }
-  GNUNET_free ((void*) argv);
-  return 0;
+  struct ClientData *cd;
+
+  cd = GNUNET_new (struct ClientData);
+  cd->client = c;
+  cd->tt = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
+                                         &force_timeout,
+                                         cd);
+  return cd;
+}
+
+
+/**
+ * Callback called when a client disconnected from the service
+ *
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls our `struct ClientData`
+ */
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *c,
+                      void *internal_cls)
+{
+  struct ClientData *cd = internal_cls;
+
+  if (NULL != cd->tt)
+    GNUNET_SCHEDULER_cancel (cd->tt);
+  GNUNET_free (cd);
 }
 
 
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+("nat-server",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_fixed_size (test,
+                          GNUNET_MESSAGE_TYPE_NAT_TEST,
+                          struct GNUNET_NAT_AUTO_TestMessage,
+                          NULL),
+ GNUNET_MQ_handler_end ());
+
+
+#if defined(LINUX) && defined(__GLIBC__)
+#include <malloc.h>
+
+/**
+ * MINIMIZE heap size (way below 128k) since this process doesn't need much.
+ */
+void __attribute__ ((constructor))
+GNUNET_ARM_memory_init ()
+{
+  mallopt (M_TRIM_THRESHOLD, 4 * 1024);
+  mallopt (M_TOP_PAD, 1 * 1024);
+  malloc_trim (0);
+}
+#endif
+
+
+
+
 /* end of gnunet-nat-server.c */
diff --git a/src/nat-auto/nat_auto_api.c b/src/nat-auto/nat_auto_api.c
index 6660239c3..8a7eaf264 100644
--- a/src/nat-auto/nat_auto_api.c
+++ b/src/nat-auto/nat_auto_api.c
@@ -224,7 +224,7 @@ GNUNET_NAT_AUTO_autoconfig_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
 
   buf = GNUNET_CONFIGURATION_serialize (cfg,
                                         &size);
-  if (size > GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*req))
+  if (size > GNUNET_MAX_MESSAGE_SIZE - sizeof (*req))
   {
     GNUNET_break (0);
     GNUNET_free (buf);
diff --git a/src/nat/gnunet-nat.c b/src/nat/gnunet-nat.c
index 80bfb5726..50e0dff0f 100644
--- a/src/nat/gnunet-nat.c
+++ b/src/nat/gnunet-nat.c
@@ -34,9 +34,9 @@
 static int global_ret;
 
 /**
- * Name of section in configuration file to use for 
+ * Name of section in configuration file to use for
  * additional options.
- */ 
+ */
 static char *section_name;
 
 /**
@@ -72,7 +72,7 @@ static char *remote_addr;
 /**
  * Should we actually bind to #bind_addr and receive and process STUN requests?
  */
-static unsigned int do_stun;
+static int do_stun;
 
 /**
  * Handle to NAT operation.
@@ -81,7 +81,7 @@ static struct GNUNET_NAT_Handle *nh;
 
 /**
  * Listen socket for STUN processing.
- */ 
+ */
 static struct GNUNET_NETWORK_Handle *ls;
 
 /**
@@ -110,7 +110,7 @@ test_finished ()
  * a function to call whenever our set of 'valid' addresses changes.
  *
  * @param cls closure, NULL
- * @param add_remove #GNUNET_YES to add a new public IP address, 
+ * @param add_remove #GNUNET_YES to add a new public IP address,
  *                   #GNUNET_NO to remove a previous (now invalid) one
  * @param ac address class the address belongs to
  * @param addr either the previous or the new public IP address
@@ -123,12 +123,12 @@ address_cb (void *cls,
             const struct sockaddr *addr,
             socklen_t addrlen)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
-              "%s %s (%d)\n",
-              add_remove ? "+" : "-",
-              GNUNET_a2s (addr,
-                          addrlen),
-              (int) ac);
+  fprintf (stdout,
+           "%s %s (%d)\n",
+           add_remove ? "+" : "-",
+           GNUNET_a2s (addr,
+                       addrlen),
+           (int) ac);
 }
 
 
@@ -186,7 +186,7 @@ static void
 stun_read_task (void *cls)
 {
   ssize_t size;
-  
+
   rtask = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
                                          ls,
                                          &stun_read_task,
@@ -204,7 +204,7 @@ stun_read_task (void *cls)
     struct sockaddr_storage sa;
     socklen_t salen = sizeof (sa);
     ssize_t ret;
-    
+
     ret = GNUNET_NETWORK_socket_recvfrom (ls,
                                           buf,
                                           size + 1,
@@ -269,6 +269,10 @@ run (void *cls,
     global_ret = 1;
     return;
   }
+  local_len = 0;
+  local_sa = NULL;
+  remote_len = 0;
+  remote_sa = NULL;
   if (NULL != local_addr)
   {
     local_len = (socklen_t) GNUNET_STRINGS_parse_socket_addr (local_addr,
@@ -279,13 +283,10 @@ run (void *cls,
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "Invalid socket address `%s'\n",
                   local_addr);
-      global_ret = 1;
-      return;
+      goto fail_and_shutdown;
     }
   }
 
-  remote_len = 0;
-
   if (NULL != remote_addr)
   {
     remote_len = GNUNET_STRINGS_parse_socket_addr (remote_addr,
@@ -296,8 +297,7 @@ run (void *cls,
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "Invalid socket address `%s'\n",
                   remote_addr);
-      global_ret = 1;
-      return;
+      goto fail_and_shutdown;
     }
   }
 
@@ -318,32 +318,26 @@ run (void *cls,
   else if (listen_reversal)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
-                "Use of `-W` only effective in combination with `-i`\n");    
-    global_ret = 1;
-    GNUNET_SCHEDULER_shutdown ();
-    return;
+                "Use of `-W` only effective in combination with `-i`\n");
+    goto fail_and_shutdown;
   }
 
   if (NULL != remote_addr)
   {
     int ret;
-    
+
     if ( (NULL == nh) ||
          (sizeof (struct sockaddr_in) != local_len) )
     {
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "Require IPv4 local address to initiate connection reversal\n");
-      global_ret = 1;
-      GNUNET_SCHEDULER_shutdown ();
-      return;
+      goto fail_and_shutdown;
     }
     if (sizeof (struct sockaddr_in) != remote_len)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "Require IPv4 reversal target address\n");
-      global_ret = 1;
-      GNUNET_SCHEDULER_shutdown ();
-      return;
+      goto fail_and_shutdown;
     }
     GNUNET_assert (AF_INET == local_sa->sa_family);
     GNUNET_assert (AF_INET == remote_sa->sa_family);
@@ -365,24 +359,20 @@ run (void *cls,
       break;
     }
   }
-  
+
   if (do_stun)
   {
     if (NULL == local_addr)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "Require local address to support STUN requests\n");
-      global_ret = 1;
-      GNUNET_SCHEDULER_shutdown ();
-      return;
+      goto fail_and_shutdown;
     }
     if (IPPROTO_UDP != proto)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
                   "STUN only supported over UDP\n");
-      global_ret = 1;
-      GNUNET_SCHEDULER_shutdown ();
-      return;
+      goto fail_and_shutdown;
     }
     ls = GNUNET_NETWORK_socket_create (af,
                                        SOCK_DGRAM,
@@ -397,17 +387,22 @@ run (void *cls,
                   GNUNET_a2s (local_sa,
                               local_len),
                   STRERROR (errno));
-      global_ret = 1;
-      GNUNET_SCHEDULER_shutdown ();
-      return;
+      goto fail_and_shutdown;
     }
     rtask = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
                                            ls,
                                            &stun_read_task,
                                            NULL);
   }
-
+  GNUNET_free_non_null (remote_sa);
+  GNUNET_free_non_null (local_sa);
   test_finished ();
+  return;
+ fail_and_shutdown:
+  global_ret = 1;
+  GNUNET_SCHEDULER_shutdown ();
+  GNUNET_free_non_null (remote_sa);
+  GNUNET_free_non_null (local_sa);
 }
 
 
@@ -422,29 +417,46 @@ int
 main (int argc,
       char *const argv[])
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'i', "in", "ADDRESS",
-     gettext_noop ("which IP and port are we locally using to bind/listen to"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &local_addr },
-    {'r', "remote", "ADDRESS",
-     gettext_noop ("which remote IP and port should be asked for connection reversal"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &remote_addr },
-    {'S', "section", NULL,
-     gettext_noop ("name of configuration section to find additional options, such as manual host punching data"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &section_name },
-    {'s', "stun", NULL,
-     gettext_noop ("enable STUN processing"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &do_stun },
-    {'t', "tcp", NULL,
-     gettext_noop ("use TCP"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &use_tcp },
-    {'u', "udp", NULL,
-     gettext_noop ("use UDP"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &use_udp },
-    {'W', "watch", NULL,
-     gettext_noop ("watch for connection reversal requests"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &listen_reversal },
-   GNUNET_GETOPT_OPTION_END
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_STRING ('i',
+                                 "in",
+                                 "ADDRESS",
+                                 gettext_noop ("which IP and port are we locally using to bind/listen to"),
+                                 &local_addr),
+
+    GNUNET_GETOPT_OPTION_STRING ('r',
+                                 "remote",
+                                 "ADDRESS",
+                                 gettext_noop ("which remote IP and port should be asked for connection reversal"),
+                                 &remote_addr),
+
+    GNUNET_GETOPT_OPTION_STRING ('S',
+                                 "section",
+                                 NULL,
+                                 gettext_noop ("name of configuration section to find additional options, such as manual host punching data"),
+                                 &section_name),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('s',
+                                  "stun",
+                                  gettext_noop ("enable STUN processing"),
+                                  &do_stun),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('t',
+                                  "tcp",
+                                  gettext_noop ("use TCP"),
+                                  &use_tcp),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('u',
+                                  "udp",
+                                  gettext_noop ("use UDP"),
+                                  &use_udp),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('W',
+                                  "watch",
+                                  gettext_noop ("watch for connection reversal requests"),
+                                  &listen_reversal),
+    GNUNET_GETOPT_OPTION_END
   };
 
   if (GNUNET_OK !=
diff --git a/src/nat/gnunet-service-nat.c b/src/nat/gnunet-service-nat.c
index bfe212308..0695c7ac7 100644
--- a/src/nat/gnunet-service-nat.c
+++ b/src/nat/gnunet-service-nat.c
@@ -662,6 +662,11 @@ notify_client (enum GNUNET_NAT_AddressClass ac,
   struct GNUNET_MQ_Envelope *env;
   struct GNUNET_NAT_AddressChangeNotificationMessage *msg;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Notifying client about %s of IP %s\n",
+              add ? "addition" : "removal",
+              GNUNET_a2s (addr,
+                          addr_len));
   env = GNUNET_MQ_msg_extra (msg,
                              addr_len,
                              GNUNET_MESSAGE_TYPE_NAT_ADDRESS_CHANGE);
@@ -693,7 +698,11 @@ check_notify_client (struct LocalAddressList *delta,
   struct sockaddr_in6 v6;
 
   if (0 == (ch->flags & GNUNET_NAT_RF_ADDRESSES))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Not notifying client as it does not care about addresses\n");
     return;
+  }
   switch (delta->af)
   {
   case AF_INET:
@@ -715,25 +724,24 @@ check_notify_client (struct LocalAddressList *delta,
            (! match_ipv4 ("127.0.0.1", &v4.sin_addr, 8)) )
         continue; /* bound to loopback, but this is not loopback */
       if ( (! match_ipv4 ("127.0.0.1", &c4->sin_addr, 8) ) &&
-           (0 != c4->sin_addr.s_addr) &&
            match_ipv4 ("127.0.0.1", &v4.sin_addr, 8) )
         continue; /* bound to non-loopback, but this is loopback */
-      if ( (0 != (ch->flags & GNUNET_NAT_AC_EXTERN)) &&
-           (0 != c4->sin_addr.s_addr) &&
-           (! is_nat_v4 (&v4.sin_addr)) )
-        continue; /* based on external-IP, but this IP is not
-                     from private address range. */
+      if ( (0 != (delta->ac & GNUNET_NAT_AC_EXTERN)) &&
+           (0 != c4->sin_addr.s_addr) &&
+           (! is_nat_v4 (&v4.sin_addr)) )
+       continue; /* based on external-IP, but this IP is not
+                    from private address range. */
       if ( (0 != memcmp (&v4.sin_addr,
-                         &c4->sin_addr,
-                         sizeof (struct in_addr))) &&
-           (0 != c4->sin_addr.s_addr) &&
-           ( (! is_nat_v4 (&c4->sin_addr)) ||
-             (0 == (ch->flags & GNUNET_NAT_AC_EXTERN))) )
+                         &c4->sin_addr,
+                         sizeof (struct in_addr))) &&
+           (0 != c4->sin_addr.s_addr) &&
+           (! is_nat_v4 (&c4->sin_addr)) )
         continue; /* this IP is not from private address range,
                      and IP does not match. */
 
       /* OK, IP seems relevant, notify client */
-      v4.sin_port = c4->sin_port;
+      if (0 == htons (v4.sin_port))
+        v4.sin_port = c4->sin_port;
       notify_client (delta->ac,
                      ch,
                      add,
@@ -760,13 +768,10 @@ check_notify_client (struct LocalAddressList *delta,
            (! match_ipv6 ("::1", &v6.sin6_addr, 128)) )
         continue; /* bound to loopback, but this is not loopback */
       if ( (! match_ipv6 ("::1", &c6->sin6_addr, 128) ) &&
-           (0 != memcmp (&c6->sin6_addr,
-                         &in6addr_any,
-                         sizeof (struct in6_addr))) &&
            match_ipv6 ("::1", &v6.sin6_addr, 128) )
         continue; /* bound to non-loopback, but this is loopback */
-      if ( (0 != (ch->flags & GNUNET_NAT_AC_EXTERN)) &&
-           (0 != memcmp (&c6->sin6_addr,
+      if ( (0 != (delta->ac & GNUNET_NAT_AC_EXTERN)) &&
+           (0 != memcmp (&c6->sin6_addr,
                          &in6addr_any,
                          sizeof (struct in6_addr))) &&
            (! is_nat_v6 (&v6.sin6_addr)) )
@@ -793,7 +798,8 @@ check_notify_client (struct LocalAddressList *delta,
                      does not match and is not an external IP */
 
       /* OK, IP seems relevant, notify client */
-      v6.sin6_port = c6->sin6_port;
+      if (0 == htons (v6.sin6_port))
+        v6.sin6_port = c6->sin6_port;
       notify_client (delta->ac,
                      ch,
                      add,
@@ -853,6 +859,10 @@ notify_client_external_ipv4_change (void *cls,
     struct LocalAddressList lal;
     struct sockaddr_in *s4;
 
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Detected eternal IP, can now back-fill AUTO:%u in hole punching specification of `%s'\n",
+                (unsigned int) ch->ext_dns_port,
+                ch->section_name);
     memset (&lal, 0, sizeof (lal));
     s4 = (struct sockaddr_in *) &lal.addr;
     s4->sin_family = AF_INET;
@@ -868,8 +878,6 @@ notify_client_external_ipv4_change (void *cls,
   /* (1) check if client cares. */
   if (! ch->natted_address)
     return;
-  if (0 == (GNUNET_NAT_RF_ADDRESSES & ch->flags))
-    return;
   have_v4 = GNUNET_NO;
   for (unsigned int i=0;i<ch->num_caddrs;i++)
   {
@@ -891,6 +899,10 @@ notify_client_external_ipv4_change (void *cls,
   sa.sin_addr = *v4;
   sa.sin_port = htons (0);
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Detected eternal IP %s, notifying client of external IP (without port)\n",
+              GNUNET_a2s ((const struct sockaddr *) &sa,
+                          sizeof (sa)));
   /* (3) notify client of change */
   notify_client (is_nat_v4 (v4)
                  ? GNUNET_NAT_AC_EXTERN | GNUNET_NAT_AC_LAN
@@ -1055,7 +1067,8 @@ run_scan (void *cls)
 
         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                     "Found NATed local address %s, starting NAT server\n",
-                    GNUNET_a2s ((void *) &pos->addr, sizeof (*s4)));
+                    GNUNET_a2s ((const struct sockaddr *) &pos->addr,
+                                sizeof (*s4)));
         pos->hc = GN_start_gnunet_nat_server_ (&s4->sin_addr,
                                                &reversal_callback,
                                                pos);
@@ -1280,6 +1293,11 @@ dyndns_lookup (void *cls)
   struct ClientHandle *ch = cls;
   struct LocalAddressList *lal;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Performing DNS lookup for punched hole given for `%s' as `%s:%u'\n",
+              ch->section_name,
+              ch->hole_external,
+              (unsigned int) ch->ext_dns_port);
   for (lal = ch->ext_addr_head; NULL != lal; lal = lal->next)
     lal->old = GNUNET_YES;
   ch->ext_dns_task = NULL;
@@ -1374,6 +1392,11 @@ lookup_hole_external (struct ClientHandle *ch)
                       ch->hole_external,
                       &s4->sin_addr))
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "IPv4 punched hole given for `%s' via `%s:%u'\n",
+                ch->section_name,
+                ch->hole_external,
+                (unsigned int) ch->ext_dns_port);
     s4->sin_port = htons (ch->ext_dns_port);
     lal->af = AF_INET;
     lal->ac = GNUNET_NAT_AC_GLOBAL | GNUNET_NAT_AC_MANUAL;
@@ -1423,8 +1446,6 @@ handle_register (void *cls,
     GNUNET_SERVICE_client_drop (ch->client);
     return;
   }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received REGISTER message from client\n");
   ch->flags = message->flags;
   ch->proto = message->proto;
   ch->num_caddrs = ntohs (message->num_addrs);
@@ -1512,6 +1533,9 @@ handle_register (void *cls,
   ch->section_name
     = GNUNET_strndup (off,
                       ntohs (message->str_len));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received REGISTER message from client for subsystem `%s'\n",
+              ch->section_name);
   if (GNUNET_OK ==
       GNUNET_CONFIGURATION_get_value_string (cfg,
                                              ch->section_name,
diff --git a/src/nat/gnunet-service-nat_externalip.c b/src/nat/gnunet-service-nat_externalip.c
index 8df6f48a5..f79ff4070 100644
--- a/src/nat/gnunet-service-nat_externalip.c
+++ b/src/nat/gnunet-service-nat_externalip.c
@@ -182,6 +182,7 @@ handle_external_ip (void *cls,
   switch (result)
   {
   case GNUNET_NAT_ERROR_SUCCESS:
+    GNUNET_assert (NULL != addr);
     if (addr->s_addr == mini_external_ipv4.s_addr)
       return; /* not change */
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
diff --git a/src/nat/gnunet-service-nat_helper.c b/src/nat/gnunet-service-nat_helper.c
index e91f63beb..de6531fa8 100644
--- a/src/nat/gnunet-service-nat_helper.c
+++ b/src/nat/gnunet-service-nat_helper.c
@@ -221,6 +221,26 @@ restart_nat_server (void *cls)
   char ia[INET_ADDRSTRLEN];
 
   h->server_read_task = NULL;
+  GNUNET_assert (NULL !=
+                 inet_ntop (AF_INET,
+                            &h->internal_address,
+                            ia,
+                            sizeof (ia)));
+  /* Start the server process */
+  binary
+    = GNUNET_OS_get_libexec_binary_path ("gnunet-helper-nat-server");
+  if (GNUNET_YES !=
+      GNUNET_OS_check_helper_binary (binary,
+                                     GNUNET_YES,
+                                     ia))
+  {
+    /* move instantly to max delay, as this is unlikely to be fixed */
+    h->server_retry_delay
+      = GNUNET_TIME_STD_EXPONENTIAL_BACKOFF_THRESHOLD;
+    GNUNET_free (binary);
+    try_again (h);
+    return;
+  }
   h->server_stdout
     = GNUNET_DISK_pipe (GNUNET_YES, GNUNET_YES,
                         GNUNET_NO, GNUNET_YES);
@@ -228,21 +248,14 @@ restart_nat_server (void *cls)
   {
     GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
                          "pipe");
+    GNUNET_free (binary);
     try_again (h);
     return;
   }
-  GNUNET_assert (NULL !=
-                 inet_ntop (AF_INET,
-                            &h->internal_address,
-                            ia,
-                            sizeof (ia)));
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Starting `%s' at `%s'\n",
               "gnunet-helper-nat-server",
               ia);
-  /* Start the server process */
-  binary
-    = GNUNET_OS_get_libexec_binary_path ("gnunet-helper-nat-server");
   h->server_proc
     = GNUNET_OS_start_process (GNUNET_NO,
                                0,
diff --git a/src/nat/nat_api.c b/src/nat/nat_api.c
index eec5d3968..69612584e 100644
--- a/src/nat/nat_api.c
+++ b/src/nat/nat_api.c
@@ -52,7 +52,7 @@ struct AddrEntry
    * Address class of the address.
    */
   enum GNUNET_NAT_AddressClass ac;
-  
+
   /**
    * Number of bytes that follow.
    */
@@ -70,7 +70,7 @@ struct GNUNET_NAT_Handle
    * Configuration we use.
    */
   const struct GNUNET_CONFIGURATION_Handle *cfg;
-  
+
   /**
    * Message queue for communicating with the NAT service.
    */
@@ -80,7 +80,7 @@ struct GNUNET_NAT_Handle
    * Our registration message.
    */
   struct GNUNET_MessageHeader *reg;
-  
+
   /**
    * Head of address DLL.
    */
@@ -95,12 +95,12 @@ struct GNUNET_NAT_Handle
    * Function to call when our addresses change.
    */
   GNUNET_NAT_AddressCallback address_callback;
-  
+
   /**
    * Function to call when another peer requests connection reversal.
    */
   GNUNET_NAT_ReversalCallback reversal_callback;
-  
+
   /**
    * Closure for the various callbacks.
    */
@@ -136,7 +136,7 @@ static void
 reconnect (struct GNUNET_NAT_Handle *nh)
 {
   struct AddrEntry *ae;
-  
+
   if (NULL != nh->mq)
   {
     GNUNET_MQ_destroy (nh->mq);
@@ -184,7 +184,7 @@ check_connection_reversal_request (void *cls,
   return GNUNET_OK;
 }
 
-  
+
 /**
  * Handle connection reversal request.
  *
@@ -247,7 +247,7 @@ check_address_change_notification (void *cls,
   return GNUNET_OK;
 }
 
-  
+
 /**
  * Handle connection reversal request.
  *
@@ -264,6 +264,8 @@ handle_address_change_notification (void *cls,
   enum GNUNET_NAT_AddressClass ac;
   struct AddrEntry *ae;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received address change notification\n");
   ac = (enum GNUNET_NAT_AddressClass) ntohl (acn->addr_class);
   if (GNUNET_YES == ntohl (acn->add_remove))
   {
@@ -395,13 +397,13 @@ GNUNET_NAT_register (const struct GNUNET_CONFIGURATION_Handle *cfg,
   size_t len;
   size_t str_len;
   char *off;
-  
+
   len = 0;
   for (unsigned int i=0;i<num_addrs;i++)
     len += addrlens[i];
   str_len = strlen (config_section) + 1;
   len += str_len;
-  if ( (len > GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*rm)) ||
+  if ( (len > GNUNET_MAX_MESSAGE_SIZE - sizeof (*rm)) ||
        (num_addrs > UINT16_MAX) )
   {
     GNUNET_break (0);
@@ -427,6 +429,7 @@ GNUNET_NAT_register (const struct GNUNET_CONFIGURATION_Handle *cfg,
       if (sizeof (struct sockaddr_in) != addrlens[i])
       {
         GNUNET_break (0);
+        GNUNET_free (rm);
         return NULL;
       }
       break;
@@ -434,6 +437,7 @@ GNUNET_NAT_register (const struct GNUNET_CONFIGURATION_Handle *cfg,
       if (sizeof (struct sockaddr_in6) != addrlens[i])
       {
         GNUNET_break (0);
+        GNUNET_free (rm);
         return NULL;
       }
       break;
@@ -442,12 +446,14 @@ GNUNET_NAT_register (const struct GNUNET_CONFIGURATION_Handle *cfg,
       if (sizeof (struct sockaddr_un) != addrlens[i])
       {
         GNUNET_break (0);
+        GNUNET_free (rm);
         return NULL;
       }
       break;
 #endif
     default:
       GNUNET_break (0);
+      GNUNET_free (rm);
       return NULL;
     }
     GNUNET_memcpy (off,
@@ -569,7 +575,7 @@ test_stun_packet (const void *data,
  *
  * The function does some basic sanity checks on packet size and
  * content, try to extract a bit of information.
- * 
+ *
  * At the moment this only processes BIND requests, and returns the
  * externally visible address of the request to the rest of the
  * NAT logic.
@@ -663,7 +669,7 @@ GNUNET_NAT_test_address (struct GNUNET_NAT_Handle *nh,
  * @param nh handle (used for configuration)
  * @param local_sa our local address of the peer (IPv4-only)
  * @param remote_sa the remote address of the peer (IPv4-only)
- * @return #GNUNET_SYSERR on error, 
+ * @return #GNUNET_SYSERR on error,
  *         #GNUNET_NO if connection reversal is unavailable,
  *         #GNUNET_OK otherwise (presumably in progress)
  */
diff --git a/src/nat/nat_stun.h b/src/nat/nat_stun.h
index 3e4228875..8d1c2720f 100644
--- a/src/nat/nat_stun.h
+++ b/src/nat/nat_stun.h
@@ -70,7 +70,7 @@ struct stun_addr
    * Port number.
    */
   uint16_t port;
-  
+
   /**
    * IPv4 address. Should this be "struct in_addr"?
    */
@@ -79,7 +79,7 @@ struct stun_addr
 
 
 /**
- * STUN message classes 
+ * STUN message classes
  */
 enum StunClasses {
   INVALID_CLASS = 0,
@@ -186,18 +186,19 @@ stun_msg2str (int msg)
   static char result[64];
   const char *msg_class = NULL;
   const char *method = NULL;
-  int value;
+  enum StunClasses cvalue;
+  enum StunMethods mvalue;
 
-  value = decode_class (msg);
+  cvalue = decode_class (msg);
   for (unsigned int i = 0; classes[i].name; i++)
-    if (classes[i].value == value)
+    if (classes[i].value == cvalue)
     {
       msg_class = classes[i].name;
       break;
     }
-  value = decode_method (msg);
+  mvalue = decode_method (msg);
   for (unsigned int i = 0; methods[i].name; i++)
-    if (methods[i].value == value)
+    if (methods[i].value == mvalue)
     {
       method = methods[i].name;
       break;
diff --git a/src/nse/gnunet-nse-profiler.c b/src/nse/gnunet-nse-profiler.c
index 48afd9298..ea28bf610 100644
--- a/src/nse/gnunet-nse-profiler.c
+++ b/src/nse/gnunet-nse-profiler.c
@@ -129,7 +129,7 @@ static int ok;
 /**
  * Be verbose (configuration option)
  */
-static int verbose;
+static unsigned int verbose;
 
 /**
  * Name of the file with the hosts to run the test over (configuration option)
@@ -835,28 +835,47 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'C', "connections", "COUNT",
-     gettext_noop ("limit to the number of connections to NSE services, 0 for none"),
-     1, &GNUNET_GETOPT_set_uint, &connection_limit},
-    {'d', "details", "FILENAME",
-     gettext_noop ("name of the file for writing connection information and statistics"),
-     1, &GNUNET_GETOPT_set_string, &data_filename},
-    {'H', "hosts", "FILENAME",
-     gettext_noop ("name of the file with the login information for the testbed"),
-     1, &GNUNET_GETOPT_set_string, &hosts_file},
-    {'o', "output", "FILENAME",
-     gettext_noop ("name of the file for writing the main results"),
-     1, &GNUNET_GETOPT_set_string, &output_filename},
-    {'p', "peers", "NETWORKSIZESPEC",
-     gettext_noop ("Number of peers to run in each round, separated by commas"),
-     1, &GNUNET_GETOPT_set_string, &num_peer_spec},
-    {'V', "verbose", NULL,
-     gettext_noop ("be verbose (print progress information)"),
-     0, &GNUNET_GETOPT_increment_value, &verbose},
-    {'w', "wait", "DELAY",
-     gettext_noop ("delay between rounds"),
-     1, &GNUNET_GETOPT_set_relative_time, &wait_time},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_UINT ('C',
+                                   "connections",
+                                   "COUNT",
+                                   gettext_noop ("limit to the number of connections to NSE services, 0 for none"),
+                                   &connection_limit),
+    GNUNET_GETOPT_OPTION_STRING ('d',
+                                 "details",
+                                 "FILENAME",
+                                 gettext_noop ("name of the file for writing connection information and statistics"),
+                                 &data_filename),
+
+    GNUNET_GETOPT_OPTION_STRING ('H',
+                                 "hosts",
+                                 "FILENAME",
+                                 gettext_noop ("name of the file with the login information for the testbed"),
+                                 &hosts_file),
+
+    GNUNET_GETOPT_OPTION_STRING ('o',
+                                 "output",
+                                 "FILENAME",
+                                 gettext_noop ("name of the file for writing the main results"),
+                                 &output_filename),
+
+
+    GNUNET_GETOPT_OPTION_STRING ('p',
+                                 "peers",
+                                 "NETWORKSIZESPEC",
+                                 gettext_noop ("Number of peers to run in each round, separated by commas"),
+                                 &num_peer_spec),
+
+    GNUNET_GETOPT_OPTION_INCREMENT_VALUE ('V',
+                                          "verbose",
+                                          gettext_noop ("be verbose (print progress information)"),
+                                          &verbose),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('w',
+                                            "wait",
+                                            "DELAY",
+                                            gettext_noop ("delay between rounds"),
+                                            &wait_time),
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/peerinfo-tool/gnunet-peerinfo.c b/src/peerinfo-tool/gnunet-peerinfo.c
index a5907c63f..59ffe2efd 100644
--- a/src/peerinfo-tool/gnunet-peerinfo.c
+++ b/src/peerinfo-tool/gnunet-peerinfo.c
@@ -837,31 +837,48 @@ state_machine (void *cls)
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'n', "numeric", NULL,
-     gettext_noop ("don't resolve host names"),
-     0, &GNUNET_GETOPT_set_one, &no_resolve},
-    {'q', "quiet", NULL,
-     gettext_noop ("output only the identity strings"),
-     0, &GNUNET_GETOPT_set_one, &be_quiet},
-    {'f', "friends", NULL,
-     gettext_noop ("include friend-only information"),
-     0, &GNUNET_GETOPT_set_one, &include_friend_only},
-    {'s', "self", NULL,
-     gettext_noop ("output our own identity only"),
-     0, &GNUNET_GETOPT_set_one, &get_self},
-    {'i', "info", NULL,
-     gettext_noop ("list all known peers"),
-     0, &GNUNET_GETOPT_set_one, &get_info},
-    {'d', "dump-hello", NULL,
-     gettext_noop ("dump hello to file"),
-     1, &GNUNET_GETOPT_set_string, &dump_hello},
-    {'g', "get-hello", NULL,
-     gettext_noop ("also output HELLO uri(s)"),
-     0, &GNUNET_GETOPT_set_one, &get_uri},
-    {'p', "put-hello", "HELLO",
-     gettext_noop ("add given HELLO uri to the database"),
-     1, &GNUNET_GETOPT_set_string, &put_uri},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "numeric",
+                                  gettext_noop ("don't resolve host names"),
+                                  &no_resolve),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('q',
+                                  "quiet",
+                                  gettext_noop ("output only the identity strings"),
+                                  &be_quiet),
+    GNUNET_GETOPT_OPTION_SET_ONE ('f',
+                                  "friends",
+                                  gettext_noop ("include friend-only information"),
+                                  &include_friend_only),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('s',
+                                  "self",
+                                  gettext_noop ("output our own identity only"),
+                                  &get_self),
+    
+    GNUNET_GETOPT_OPTION_SET_ONE ('i',
+                                  "info",
+                                  gettext_noop ("list all known peers"),
+                                  &get_info),
+
+    GNUNET_GETOPT_OPTION_STRING ('d',
+                                 "dump-hello",
+                                 NULL,
+                                 gettext_noop ("dump hello to file"),
+                                 &dump_hello),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('g',
+                                  "get-hello",
+                                  gettext_noop ("also output HELLO uri(s)"),
+                                  &get_uri),
+
+    GNUNET_GETOPT_OPTION_STRING ('p',
+                                 "put-hello",
+                                 "HELLO",
+                                 gettext_noop ("add given HELLO uri to the database"),
+                                 &put_uri),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/peerinfo/gnunet-service-peerinfo.c b/src/peerinfo/gnunet-service-peerinfo.c
index bc4a2a10d..731c24bf1 100644
--- a/src/peerinfo/gnunet-service-peerinfo.c
+++ b/src/peerinfo/gnunet-service-peerinfo.c
@@ -283,7 +283,7 @@ read_host_file (const char *fn,
                 int unlink_garbage,
                 struct ReadHostFileContext *r)
 {
-  char buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1] GNUNET_ALIGN;
+  char buffer[GNUNET_MAX_MESSAGE_SIZE - 1] GNUNET_ALIGN;
   ssize_t size_total;
   struct GNUNET_TIME_Absolute now;
   unsigned int left;
@@ -919,7 +919,7 @@ add_to_tc (void *cls,
   {
         /* Copy public HELLO */
     hs = GNUNET_HELLO_size (pos->hello);
-    GNUNET_assert (hs < GNUNET_SERVER_MAX_MESSAGE_SIZE -
+    GNUNET_assert (hs < GNUNET_MAX_MESSAGE_SIZE -
                    sizeof (struct InfoMessage));
     env = GNUNET_MQ_msg_extra (im,
                                hs,
@@ -937,7 +937,7 @@ add_to_tc (void *cls,
   {
         /* Copy friend only HELLO */
     hs = GNUNET_HELLO_size (pos->friend_only_hello);
-    GNUNET_assert (hs < GNUNET_SERVER_MAX_MESSAGE_SIZE -
+    GNUNET_assert (hs < GNUNET_MAX_MESSAGE_SIZE -
                    sizeof (struct InfoMessage));
     env = GNUNET_MQ_msg_extra (im,
                                hs,
@@ -977,7 +977,7 @@ discard_hosts_helper (void *cls,
                       const char *fn)
 {
   struct GNUNET_TIME_Absolute *now = cls;
-  char buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1] GNUNET_ALIGN;
+  char buffer[GNUNET_MAX_MESSAGE_SIZE - 1] GNUNET_ALIGN;
   const struct GNUNET_HELLO_Message *hello;
   struct GNUNET_HELLO_Message *new_hello;
   int read_size;
diff --git a/src/peerstore/Makefile.am b/src/peerstore/Makefile.am
index 21db6ad17..3aef05769 100644
--- a/src/peerstore/Makefile.am
+++ b/src/peerstore/Makefile.am
@@ -73,7 +73,9 @@ libgnunet_plugin_peerstore_sqlite_la_SOURCES = \
   plugin_peerstore_sqlite.c
 libgnunet_plugin_peerstore_sqlite_la_LIBADD = \
   libgnunetpeerstore.la  \
-  $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 \
+  $(top_builddir)/src/sq/libgnunetsq.la \
+  $(top_builddir)/src/util/libgnunetutil.la \
+  $(XLIBS) -lsqlite3 \
   $(LTLIBINTL)
 libgnunet_plugin_peerstore_sqlite_la_LDFLAGS = \
  $(GN_PLUGIN_LDFLAGS)
diff --git a/src/peerstore/gnunet-service-peerstore.c b/src/peerstore/gnunet-service-peerstore.c
index 8200c2366..92d020799 100644
--- a/src/peerstore/gnunet-service-peerstore.c
+++ b/src/peerstore/gnunet-service-peerstore.c
@@ -260,15 +260,23 @@ record_iterator (void *cls,
     GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (cls_record->client),
                     env);
     if (NULL == emsg)
+    {
       GNUNET_SERVICE_client_continue (cls_record->client);
+    }
     else
+    {
+      GNUNET_break (0);
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Failed to iterate: %s\n",
+                  emsg);
       GNUNET_SERVICE_client_drop (cls_record->client);
+    }
     PEERSTORE_destroy_record (cls_record);
     return;
   }
 
   env = PEERSTORE_create_record_mq_envelope (record->sub_system,
-                                             record->peer,
+                                             &record->peer,
                                              record->key,
                                              record->value,
                                              record->value_size,
@@ -301,7 +309,7 @@ watch_notifier_it (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Found a watcher to update.\n");
   env = PEERSTORE_create_record_mq_envelope (record->sub_system,
-                                             record->peer,
+                                             &record->peer,
                                              record->key,
                                              record->value,
                                              record->value_size,
@@ -325,7 +333,7 @@ watch_notifier (struct GNUNET_PEERSTORE_Record *record)
   struct GNUNET_HashCode keyhash;
 
   PEERSTORE_hash_key (record->sub_system,
-                      record->peer,
+                      &record->peer,
                       record->key,
                       &keyhash);
   GNUNET_CONTAINER_multihashmap_get_multiple (watchers,
@@ -434,17 +442,18 @@ handle_iterate (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Iterate request: ss `%s', peer `%s', key `%s'\n",
               record->sub_system,
-              (NULL == record->peer) ? "NULL" : GNUNET_i2s (record->peer),
+              GNUNET_i2s (&record->peer),
               (NULL == record->key) ? "NULL" : record->key);
   record->client = client;
   if (GNUNET_OK !=
       db->iterate_records (db->cls,
                            record->sub_system,
-                           record->peer,
+                           (ntohs (srm->peer_set)) ? &record->peer : NULL,
                            record->key,
                            &record_iterator,
                            record))
   {
+    GNUNET_break (0);
     GNUNET_SERVICE_client_drop (client);
     PEERSTORE_destroy_record (record);
   }
@@ -470,6 +479,7 @@ store_record_continuation (void *cls,
   }
   else
   {
+    GNUNET_break (0);
     GNUNET_SERVICE_client_drop (record->client);
   }
   PEERSTORE_destroy_record (record);
@@ -496,7 +506,6 @@ check_store (void *cls,
     return GNUNET_SYSERR;
   }
   if ( (NULL == record->sub_system) ||
-       (NULL == record->peer) ||
        (NULL == record->key) )
   {
     GNUNET_break (0);
@@ -525,22 +534,23 @@ handle_store (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Received a store request. Sub system `%s' Peer `%s Key `%s' Options: %u.\n",
               record->sub_system,
-              GNUNET_i2s (record->peer),
+              GNUNET_i2s (&record->peer),
               record->key,
               (uint32_t) ntohl (srm->options));
   record->client = client;
   if (GNUNET_OK !=
       db->store_record (db->cls,
                         record->sub_system,
-                        record->peer,
+                        &record->peer,
                         record->key,
                         record->value,
                         record->value_size,
-                        *record->expiry,
+                        record->expiry,
                         ntohl (srm->options),
                         &store_record_continuation,
                         record))
   {
+    GNUNET_break (0);
     PEERSTORE_destroy_record (record);
     GNUNET_SERVICE_client_drop (client);
     return;
diff --git a/src/peerstore/peerstore.h b/src/peerstore/peerstore.h
index 8b3c4dd92..c6b954676 100644
--- a/src/peerstore/peerstore.h
+++ b/src/peerstore/peerstore.h
@@ -60,7 +60,7 @@ struct StoreRecordMessage
   /**
    * Expiry time of entry
    */
-  struct GNUNET_TIME_Absolute expiry GNUNET_PACKED;
+  struct GNUNET_TIME_AbsoluteNBO expiry;
 
   /**
    * Size of the key string
diff --git a/src/peerstore/peerstore_api.c b/src/peerstore/peerstore_api.c
index c9fcd17ab..df182fe10 100644
--- a/src/peerstore/peerstore_api.c
+++ b/src/peerstore/peerstore_api.c
@@ -286,6 +286,10 @@ store_request_sent (void *cls)
 /*******************         CONNECTION FUNCTIONS         *********************/
 /******************************************************************************/
 
+
+/**
+ * Function called when we had trouble talking to the service.
+ */
 static void
 handle_client_error (void *cls,
                      enum GNUNET_MQ_Error error)
@@ -293,7 +297,7 @@ handle_client_error (void *cls,
   struct GNUNET_PEERSTORE_Handle *h = cls;
 
   LOG (GNUNET_ERROR_TYPE_ERROR,
-       _("Received an error notification from MQ of type: %d\n"),
+       "Received an error notification from MQ of type: %d\n",
        error);
   reconnect (h);
 }
@@ -341,7 +345,9 @@ iterate_timeout (void *cls)
   callback_cls = ic->callback_cls;
   GNUNET_PEERSTORE_iterate_cancel (ic);
   if (NULL != callback)
-    callback (callback_cls, NULL, _("timeout"));
+    callback (callback_cls,
+              NULL,
+              _("timeout"));
 }
 
 
@@ -510,7 +516,7 @@ GNUNET_PEERSTORE_store (struct GNUNET_PEERSTORE_Handle *h,
        "Storing value (size: %lu) for subsytem `%s', peer `%s', key `%s'\n",
        size, sub_system, GNUNET_i2s (peer), key);
   ev = PEERSTORE_create_record_mq_envelope (sub_system, peer, key, value, size,
-                                            &expiry, options,
+                                            expiry, options,
                                             GNUNET_MESSAGE_TYPE_PEERSTORE_STORE);
   sc = GNUNET_new (struct GNUNET_PEERSTORE_StoreContext);
 
@@ -684,8 +690,12 @@ GNUNET_PEERSTORE_iterate (struct GNUNET_PEERSTORE_Handle *h,
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_PEERSTORE_IterateContext *ic;
 
-  ev = PEERSTORE_create_record_mq_envelope (sub_system, peer, key, NULL, 0,
+  ev = PEERSTORE_create_record_mq_envelope (sub_system,
+                                            peer,
+                                            key,
                                             NULL, 0,
+                                            GNUNET_TIME_UNIT_FOREVER_ABS,
+                                            0,
                                             GNUNET_MESSAGE_TYPE_PEERSTORE_ITERATE);
   ic = GNUNET_new (struct GNUNET_PEERSTORE_IterateContext);
 
@@ -756,7 +766,7 @@ handle_watch_record (void *cls,
     return;
   }
   PEERSTORE_hash_key (record->sub_system,
-                      record->peer,
+                      &record->peer,
                       record->key,
                       &keyhash);
   // FIXME: what if there are multiple watches for the same key?
@@ -848,9 +858,12 @@ reconnect (struct GNUNET_PEERSTORE_Handle *h)
                                               &ic->peer,
                                               ic->key,
                                               NULL, 0,
-                                              NULL, 0,
+                                              GNUNET_TIME_UNIT_FOREVER_ABS,
+                                              0,
                                               GNUNET_MESSAGE_TYPE_PEERSTORE_ITERATE);
     GNUNET_MQ_send (h->mq, ev);
+    if (NULL != ic->timeout_task)
+      GNUNET_SCHEDULER_cancel (ic->timeout_task);
     ic->timeout_task
       = GNUNET_SCHEDULER_add_delayed (ic->timeout,
                                       &iterate_timeout,
@@ -863,7 +876,7 @@ reconnect (struct GNUNET_PEERSTORE_Handle *h)
                                               sc->key,
                                               sc->value,
                                               sc->size,
-                                              &sc->expiry,
+                                              sc->expiry,
                                               sc->options,
                                               GNUNET_MESSAGE_TYPE_PEERSTORE_STORE);
     GNUNET_MQ_notify_sent (ev,
diff --git a/src/peerstore/peerstore_common.c b/src/peerstore/peerstore_common.c
index d12c4e21e..e0ab778fa 100644
--- a/src/peerstore/peerstore_common.c
+++ b/src/peerstore/peerstore_common.c
@@ -77,7 +77,7 @@ PEERSTORE_create_record_mq_envelope (const char *sub_system,
                                      const char *key,
                                      const void *value,
                                      size_t value_size,
-                                     struct GNUNET_TIME_Absolute *expiry,
+                                     struct GNUNET_TIME_Absolute expiry,
                                      enum GNUNET_PEERSTORE_StoreOption options,
                                      uint16_t msg_type)
 {
@@ -97,8 +97,7 @@ PEERSTORE_create_record_mq_envelope (const char *sub_system,
   msg_size = ss_size + key_size + value_size;
   ev = GNUNET_MQ_msg_extra (srm, msg_size, msg_type);
   srm->key_size = htons (key_size);
-  if (NULL != expiry)
-    srm->expiry = *expiry;
+  srm->expiry = GNUNET_TIME_absolute_hton (expiry);
   if (NULL == peer)
     srm->peer_set = htons (GNUNET_NO);
   else
@@ -147,12 +146,9 @@ PEERSTORE_parse_record_message (const struct StoreRecordMessage *srm)
   record = GNUNET_new (struct GNUNET_PEERSTORE_Record);
   if (GNUNET_YES == ntohs (srm->peer_set))
   {
-    record->peer = GNUNET_new (struct GNUNET_PeerIdentity);
-    *record->peer = srm->peer;
+    record->peer = srm->peer;
   }
-  record->expiry = GNUNET_new (struct GNUNET_TIME_Absolute);
-
-  *(record->expiry) = srm->expiry;
+  record->expiry = GNUNET_TIME_absolute_ntoh (srm->expiry);
   dummy = (char *) &srm[1];
   if (ss_size > 0)
   {
@@ -167,7 +163,9 @@ PEERSTORE_parse_record_message (const struct StoreRecordMessage *srm)
   if (value_size > 0)
   {
     record->value = GNUNET_malloc (value_size);
-    GNUNET_memcpy (record->value, dummy, value_size);
+    GNUNET_memcpy (record->value,
+                   dummy,
+                   value_size);
   }
   record->value_size = value_size;
   return record;
@@ -184,8 +182,6 @@ PEERSTORE_destroy_record (struct GNUNET_PEERSTORE_Record *record)
 {
   if (NULL != record->sub_system)
     GNUNET_free (record->sub_system);
-  if (NULL != record->peer)
-    GNUNET_free (record->peer);
   if (NULL != record->key)
     GNUNET_free (record->key);
   if (NULL != record->value)
@@ -193,7 +189,5 @@ PEERSTORE_destroy_record (struct GNUNET_PEERSTORE_Record *record)
     GNUNET_free (record->value);
     record->value = 0;
   }
-  if (NULL != record->expiry)
-    GNUNET_free (record->expiry);
   GNUNET_free (record);
 }
diff --git a/src/peerstore/peerstore_common.h b/src/peerstore/peerstore_common.h
index 3d938b5da..0fc14d9b4 100644
--- a/src/peerstore/peerstore_common.h
+++ b/src/peerstore/peerstore_common.h
@@ -56,7 +56,7 @@ PEERSTORE_create_record_mq_envelope (const char *sub_system,
                                      const char *key,
                                      const void *value,
                                      size_t value_size,
-                                     struct GNUNET_TIME_Absolute *expiry,
+                                     struct GNUNET_TIME_Absolute expiry,
                                      enum GNUNET_PEERSTORE_StoreOption options,
                                      uint16_t msg_type);
 
diff --git a/src/peerstore/plugin_peerstore_flat.c b/src/peerstore/plugin_peerstore_flat.c
index 2b90719d9..c75b2b1e4 100644
--- a/src/peerstore/plugin_peerstore_flat.c
+++ b/src/peerstore/plugin_peerstore_flat.c
@@ -107,7 +107,9 @@ delete_entries (void *cls,
   struct GNUNET_PEERSTORE_Record *entry = value;
   if (0 != strcmp (plugin->iter_key, entry->key))
     return GNUNET_YES;
-  if (0 != memcmp (plugin->iter_peer, entry->peer, sizeof (struct GNUNET_PeerIdentity)))
+  if (0 != memcmp (plugin->iter_peer,
+                   &entry->peer,
+                   sizeof (struct GNUNET_PeerIdentity)))
     return GNUNET_YES;
   if (0 != strcmp (plugin->iter_sub_system, entry->sub_system))
     return GNUNET_YES;
@@ -153,7 +155,7 @@ expire_entries (void *cls,
   struct Plugin *plugin = cls;
   struct GNUNET_PEERSTORE_Record *entry = value;
 
-  if (entry->expiry->abs_value_us < plugin->iter_now.abs_value_us)
+  if (entry->expiry.abs_value_us < plugin->iter_now.abs_value_us)
   {
     GNUNET_CONTAINER_multihashmap_remove (plugin->hm, key, value);
     plugin->exp_changes++;
@@ -204,7 +206,7 @@ iterate_entries (void *cls,
 
   if ((NULL != plugin->iter_peer) &&
       (0 != memcmp (plugin->iter_peer,
-                    entry->peer,
+                    &entry->peer,
                     sizeof (struct GNUNET_PeerIdentity))))
   {
     return GNUNET_YES;
@@ -296,10 +298,8 @@ peerstore_flat_store_record (void *cls, const char *sub_system,
   entry->value = GNUNET_malloc (size);
   GNUNET_memcpy (entry->value, value, size);
   entry->value_size = size;
-  entry->peer = GNUNET_new (struct GNUNET_PeerIdentity);
-  GNUNET_memcpy (entry->peer, peer, sizeof (struct GNUNET_PeerIdentity));
-  entry->expiry = GNUNET_new (struct GNUNET_TIME_Absolute);
-  entry->expiry->abs_value_us = expiry.abs_value_us;
+  entry->peer = *peer;
+  entry->expiry = expiry;
 
   peer_id = GNUNET_i2s (peer);
   GNUNET_CRYPTO_hash (peer_id,
@@ -409,7 +409,7 @@ database_setup (struct Plugin *plugin)
     GNUNET_free (buffer);
     return GNUNET_SYSERR;
   }
-  
+
   buffer[size] = '\0';
   GNUNET_DISK_file_close (fh);
   if (0 < size) {
@@ -433,22 +433,35 @@ database_setup (struct Plugin *plugin)
       entry = GNUNET_new (struct GNUNET_PEERSTORE_Record);
       entry->sub_system = GNUNET_strdup (sub_system);
       entry->key = GNUNET_strdup (key);
-      GNUNET_STRINGS_base64_decode (peer,
-                                    strlen (peer),
-                                    (char**)&entry->peer);
+      {
+        size_t s;
+        char *o;
+
+        o = NULL;
+        s = GNUNET_STRINGS_base64_decode (peer,
+                                          strlen (peer),
+                                          &o);
+        if (sizeof (struct GNUNET_PeerIdentity) == s)
+          GNUNET_memcpy (&entry->peer,
+                         o,
+                         s);
+        else
+          GNUNET_break (0);
+        GNUNET_free_non_null (o);
+      }
       entry->value_size = GNUNET_STRINGS_base64_decode (value,
                                                         strlen (value),
                                                         (char**)&entry->value);
-      if (GNUNET_SYSERR == GNUNET_STRINGS_fancy_time_to_absolute (expiry,
-                                             entry->expiry))
+      if (GNUNET_SYSERR ==
+          GNUNET_STRINGS_fancy_time_to_absolute (expiry,
+                                                 &entry->expiry))
       {
         GNUNET_free (entry->sub_system);
         GNUNET_free (entry->key);
-        GNUNET_free (entry->peer);
         GNUNET_free (entry);
         break;
       }
-      peer_id = GNUNET_i2s (entry->peer);
+      peer_id = GNUNET_i2s (&entry->peer);
       GNUNET_CRYPTO_hash (peer_id,
                           strlen (peer_id),
                           &hkey);
@@ -479,8 +492,8 @@ store_and_free_entries (void *cls,
   GNUNET_STRINGS_base64_encode (entry->value,
                                 entry->value_size,
                                 &val);
-  expiry = GNUNET_STRINGS_absolute_time_to_string (*entry->expiry);
-  GNUNET_STRINGS_base64_encode ((char*)entry->peer,
+  expiry = GNUNET_STRINGS_absolute_time_to_string (entry->expiry);
+  GNUNET_STRINGS_base64_encode ((char*)&entry->peer,
                                 sizeof (struct GNUNET_PeerIdentity),
                                 &peer);
   GNUNET_asprintf (&line,
@@ -496,10 +509,8 @@ store_and_free_entries (void *cls,
                           line,
                           strlen (line));
   GNUNET_free (entry->sub_system);
-  GNUNET_free (entry->peer);
   GNUNET_free (entry->key);
   GNUNET_free (entry->value);
-  GNUNET_free (entry->expiry);
   GNUNET_free (entry);
   GNUNET_free (line);
   return GNUNET_YES;
diff --git a/src/peerstore/plugin_peerstore_sqlite.c b/src/peerstore/plugin_peerstore_sqlite.c
index 2cd7e22cf..440263d44 100644
--- a/src/peerstore/plugin_peerstore_sqlite.c
+++ b/src/peerstore/plugin_peerstore_sqlite.c
@@ -1,6 +1,6 @@
 /*
  * This file is part of GNUnet
- * Copyright (C) 2013 GNUnet e.V.
+ * Copyright (C) 2013, 2017 GNUnet e.V.
  *
  * GNUnet is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published
@@ -22,11 +22,13 @@
  * @file peerstore/plugin_peerstore_sqlite.c
  * @brief sqlite-based peerstore backend
  * @author Omar Tarabai
+ * @author Christian Grothoff
  */
 
 #include "platform.h"
 #include "gnunet_peerstore_plugin.h"
 #include "gnunet_peerstore_service.h"
+#include "gnunet_sq_lib.h"
 #include "peerstore.h"
 #include <sqlite3.h>
 
@@ -111,6 +113,7 @@ struct Plugin
 
 };
 
+
 /**
  * Delete records with the given key
  *
@@ -118,40 +121,50 @@ struct Plugin
  * @param sub_system name of sub system
  * @param peer Peer identity (can be NULL)
  * @param key entry key string (can be NULL)
- * @return number of deleted records
+ * @return number of deleted records, #GNUNE_SYSERR on error
  */
 static int
-peerstore_sqlite_delete_records (void *cls, const char *sub_system,
+peerstore_sqlite_delete_records (void *cls,
+                                 const char *sub_system,
                                  const struct GNUNET_PeerIdentity *peer,
                                  const char *key)
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt = plugin->delete_peerstoredata;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_string (sub_system),
+    GNUNET_SQ_query_param_auto_from_type (peer),
+    GNUNET_SQ_query_param_string (key),
+    GNUNET_SQ_query_param_end
+  };
+  int ret;
 
-  if ((SQLITE_OK !=
-       sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                          SQLITE_STATIC)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_blob (stmt, 2, peer, sizeof (struct GNUNET_PeerIdentity),
-                          SQLITE_STATIC)) ||
-      (SQLITE_OK !=
-       sqlite3_bind_text (stmt, 3, key, strlen (key) + 1, SQLITE_STATIC)))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (stmt,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
+    return GNUNET_SYSERR;
   }
-  else if (SQLITE_DONE != sqlite3_step (stmt))
+  if (SQLITE_DONE !=
+      sqlite3_step (stmt))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
+    ret = GNUNET_SYSERR;
   }
-  if (SQLITE_OK != sqlite3_reset (stmt))
+  else
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
-    return 0;
+    ret = sqlite3_changes (plugin->dbh);
   }
-  return sqlite3_changes (plugin->dbh);
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
+  return ret;
 }
 
 
@@ -172,28 +185,36 @@ peerstore_sqlite_expire_records (void *cls, struct GNUNET_TIME_Absolute now,
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt = plugin->expire_peerstoredata;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_absolute_time (&now),
+    GNUNET_SQ_query_param_end
+  };
 
-  if (SQLITE_OK !=
-      sqlite3_bind_int64 (stmt, 1, (sqlite3_uint64) now.abs_value_us))
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (stmt,
+                      params))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
+    return GNUNET_SYSERR;
   }
-  else if (SQLITE_DONE != sqlite3_step (stmt))
+  if (SQLITE_DONE != sqlite3_step (stmt))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
-  }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
     return GNUNET_SYSERR;
   }
   if (NULL != cont)
-  {
-    cont (cont_cls, sqlite3_changes (plugin->dbh));
-  }
+    cont (cont_cls,
+          sqlite3_changes (plugin->dbh));
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   return GNUNET_OK;
 }
 
@@ -213,7 +234,8 @@ peerstore_sqlite_expire_records (void *cls, struct GNUNET_TIME_Absolute now,
  * called
  */
 static int
-peerstore_sqlite_iterate_records (void *cls, const char *sub_system,
+peerstore_sqlite_iterate_records (void *cls,
+                                  const char *sub_system,
                                   const struct GNUNET_PeerIdentity *peer,
                                   const char *key,
                                   GNUNET_PEERSTORE_Processor iter,
@@ -223,94 +245,115 @@ peerstore_sqlite_iterate_records (void *cls, const char *sub_system,
   sqlite3_stmt *stmt;
   int err = 0;
   int sret;
-  struct GNUNET_PEERSTORE_Record *ret;
+  struct GNUNET_PEERSTORE_Record rec;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Executing iterate request on sqlite db.\n");
-  if (NULL == peer && NULL == key)
-  {
-    stmt = plugin->select_peerstoredata;
-    err =
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                            SQLITE_STATIC));
-  }
-  else if (NULL == key)
-  {
-    stmt = plugin->select_peerstoredata_by_pid;
-    err =
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                            SQLITE_STATIC)) ||
-        (SQLITE_OK !=
-         sqlite3_bind_blob (stmt, 2, peer, sizeof (struct GNUNET_PeerIdentity),
-                            SQLITE_STATIC));
-  }
-  else if (NULL == peer)
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Executing iterate request on sqlite db.\n");
+  if (NULL == peer)
   {
-    stmt = plugin->select_peerstoredata_by_key;
-    err =
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                            SQLITE_STATIC)) ||
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 2, key, strlen (key) + 1, SQLITE_STATIC));
+    if (NULL == key)
+    {
+      struct GNUNET_SQ_QueryParam params[] = {
+        GNUNET_SQ_query_param_string (sub_system),
+        GNUNET_SQ_query_param_end
+      };
+
+      stmt = plugin->select_peerstoredata;
+      err = GNUNET_SQ_bind (stmt,
+                            params);
+    }
+    else
+    {
+      struct GNUNET_SQ_QueryParam params[] = {
+        GNUNET_SQ_query_param_string (sub_system),
+        GNUNET_SQ_query_param_string (key),
+        GNUNET_SQ_query_param_end
+      };
+
+      stmt = plugin->select_peerstoredata_by_key;
+      err = GNUNET_SQ_bind (stmt,
+                            params);
+    }
   }
   else
   {
-    stmt = plugin->select_peerstoredata_by_all;
-    err =
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                            SQLITE_STATIC)) ||
-        (SQLITE_OK !=
-         sqlite3_bind_blob (stmt, 2, peer, sizeof (struct GNUNET_PeerIdentity),
-                            SQLITE_STATIC)) ||
-        (SQLITE_OK !=
-         sqlite3_bind_text (stmt, 3, key, strlen (key) + 1, SQLITE_STATIC));
+    if (NULL == key)
+    {
+      struct GNUNET_SQ_QueryParam params[] = {
+        GNUNET_SQ_query_param_string (sub_system),
+        GNUNET_SQ_query_param_auto_from_type (peer),
+        GNUNET_SQ_query_param_end
+      };
+
+      stmt = plugin->select_peerstoredata_by_pid;
+      err = GNUNET_SQ_bind (stmt,
+                            params);
+    }
+    else
+    {
+      struct GNUNET_SQ_QueryParam params[] = {
+        GNUNET_SQ_query_param_string (sub_system),
+        GNUNET_SQ_query_param_auto_from_type (peer),
+        GNUNET_SQ_query_param_string (key),
+        GNUNET_SQ_query_param_end
+      };
+
+      stmt = plugin->select_peerstoredata_by_all;
+      err = GNUNET_SQ_bind (stmt,
+                            params);
+    }
   }
 
-  if (err)
+  if (GNUNET_OK != err)
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind_XXXX");
-    if (SQLITE_OK != sqlite3_reset (stmt))
-      LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                  "sqlite3_reset");
+    GNUNET_SQ_reset (plugin->dbh,
+                     stmt);
     return GNUNET_SYSERR;
   }
+
+  err = 0;
   while (SQLITE_ROW == (sret = sqlite3_step (stmt)))
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Returning a matched record.\n");
-    ret = GNUNET_new (struct GNUNET_PEERSTORE_Record);
-
-    ret->sub_system = (char *) sqlite3_column_text (stmt, 0);
-    ret->peer = (struct GNUNET_PeerIdentity *) sqlite3_column_blob (stmt, 1);
-    ret->key = (char *) sqlite3_column_text (stmt, 2);
-    ret->value = (void *) sqlite3_column_blob (stmt, 3);
-    ret->value_size = sqlite3_column_bytes (stmt, 3);
-    ret->expiry = GNUNET_new (struct GNUNET_TIME_Absolute);
-
-    ret->expiry->abs_value_us = (uint64_t) sqlite3_column_int64 (stmt, 4);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Returning a matched record.\n");
+    struct GNUNET_SQ_ResultSpec rs[] = {
+      GNUNET_SQ_result_spec_string (&rec.sub_system),
+      GNUNET_SQ_result_spec_auto_from_type (&rec.peer),
+      GNUNET_SQ_result_spec_string (&rec.key),
+      GNUNET_SQ_result_spec_variable_size (&rec.value, &rec.value_size),
+      GNUNET_SQ_result_spec_absolute_time (&rec.expiry),
+      GNUNET_SQ_result_spec_end
+    };
+
+    if (GNUNET_OK !=
+        GNUNET_SQ_extract_result (stmt,
+                                  rs))
+    {
+      GNUNET_break (0);
+      break;
+    }
     if (NULL != iter)
-      iter (iter_cls, ret, NULL);
-    GNUNET_free (ret->expiry);
-    GNUNET_free (ret);
+      iter (iter_cls,
+            &rec,
+            NULL);
+    GNUNET_SQ_cleanup_result (rs);
   }
   if (SQLITE_DONE != sret)
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
-    err = 1;
-  }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite_step");
     err = 1;
   }
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   if (NULL != iter)
-  {
-    iter (iter_cls, NULL, err ? "sqlite error" : NULL);
-  }
+    iter (iter_cls,
+          NULL,
+          err ? "sqlite error" : NULL);
   return GNUNET_OK;
 }
 
@@ -333,9 +376,12 @@ peerstore_sqlite_iterate_records (void *cls, const char *sub_system,
  * @return #GNUNET_OK on success, else #GNUNET_SYSERR and cont is not called
  */
 static int
-peerstore_sqlite_store_record (void *cls, const char *sub_system,
+peerstore_sqlite_store_record (void *cls,
+                               const char *sub_system,
                                const struct GNUNET_PeerIdentity *peer,
-                               const char *key, const void *value, size_t size,
+                               const char *key,
+                               const void *value,
+                               size_t size,
                                struct GNUNET_TIME_Absolute expiry,
                                enum GNUNET_PEERSTORE_StoreOption options,
                                GNUNET_PEERSTORE_Continuation cont,
@@ -343,39 +389,39 @@ peerstore_sqlite_store_record (void *cls, const char *sub_system,
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt = plugin->insert_peerstoredata;
+  struct GNUNET_SQ_QueryParam params[] = {
+    GNUNET_SQ_query_param_string (sub_system),
+    GNUNET_SQ_query_param_auto_from_type (peer),
+    GNUNET_SQ_query_param_string (key),
+    GNUNET_SQ_query_param_fixed_size (value, size),
+    GNUNET_SQ_query_param_absolute_time (&expiry),
+    GNUNET_SQ_query_param_end
+  };
 
   if (GNUNET_PEERSTORE_STOREOPTION_REPLACE == options)
   {
-    peerstore_sqlite_delete_records (cls, sub_system, peer, key);
+    peerstore_sqlite_delete_records (cls,
+                                     sub_system,
+                                     peer,
+                                     key);
   }
-  if (SQLITE_OK !=
-      sqlite3_bind_text (stmt, 1, sub_system, strlen (sub_system) + 1,
-                         SQLITE_STATIC) ||
-      SQLITE_OK != sqlite3_bind_blob (stmt, 2, peer,
-                                      sizeof (struct GNUNET_PeerIdentity),
-                                      SQLITE_STATIC) ||
-      SQLITE_OK != sqlite3_bind_text (stmt, 3, key, strlen (key) + 1,
-                                      SQLITE_STATIC) ||
-      SQLITE_OK != sqlite3_bind_blob (stmt, 4, value, size, SQLITE_STATIC) ||
-      SQLITE_OK != sqlite3_bind_int64 (stmt, 5,
-                                       (sqlite3_uint64) expiry.abs_value_us))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+  if (GNUNET_OK !=
+      GNUNET_SQ_bind (stmt,
+                      params))
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_bind");
   else if (SQLITE_DONE != sqlite3_step (stmt))
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
                 "sqlite3_step");
   }
-  if (SQLITE_OK != sqlite3_reset (stmt))
-  {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "sqlite3_reset");
-    return GNUNET_SYSERR;
-  }
+  GNUNET_SQ_reset (plugin->dbh,
+                   stmt);
   if (NULL != cont)
-  {
-    cont (cont_cls, GNUNET_OK);
-  }
+    cont (cont_cls,
+          GNUNET_OK);
   return GNUNET_OK;
 }
 
@@ -388,15 +434,25 @@ peerstore_sqlite_store_record (void *cls, const char *sub_system,
  * @return 0 on success
  */
 static int
-sql_exec (sqlite3 * dbh, const char *sql)
+sql_exec (sqlite3 *dbh,
+          const char *sql)
 {
   int result;
 
-  result = sqlite3_exec (dbh, sql, NULL, NULL, NULL);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Executed `%s' / %d\n", sql, result);
-  if (result != SQLITE_OK)
-    LOG (GNUNET_ERROR_TYPE_ERROR, _("Error executing SQL query: %s\n  %s\n"),
-         sqlite3_errmsg (dbh), sql);
+  result = sqlite3_exec (dbh,
+                         sql,
+                         NULL,
+                         NULL,
+                         NULL);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Executed `%s' / %d\n",
+       sql,
+       result);
+  if (SQLITE_OK != result)
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Error executing SQL query: %s\n  %s\n"),
+         sqlite3_errmsg (dbh),
+         sql);
   return result;
 }
 
@@ -410,38 +466,33 @@ sql_exec (sqlite3 * dbh, const char *sql)
  * @return 0 on success
  */
 static int
-sql_prepare (sqlite3 * dbh, const char *sql, sqlite3_stmt ** stmt)
+sql_prepare (sqlite3 *dbh,
+             const char *sql,
+             sqlite3_stmt ** stmt)
 {
   char *tail;
   int result;
 
-  result =
-      sqlite3_prepare_v2 (dbh, sql, strlen (sql), stmt, (const char **) &tail);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Prepared `%s' / %p: %d\n", sql, *stmt, result);
-  if (result != SQLITE_OK)
-    LOG (GNUNET_ERROR_TYPE_ERROR, _("Error preparing SQL query: %s\n  %s\n"),
-         sqlite3_errmsg (dbh), sql);
+  result = sqlite3_prepare_v2 (dbh,
+                               sql,
+                               strlen (sql),
+                               stmt,
+                               (const char **) &tail);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Prepared `%s' / %p: %d\n",
+       sql,
+       *stmt,
+       result);
+  if (SQLITE_OK != result)
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Error preparing SQL query: %s\n  %s\n"),
+         sqlite3_errmsg (dbh),
+         sql);
   return result;
 }
 
 
 /**
- * sqlite3 custom function for comparison of uint64_t values
- * since it is not supported by default
- */
-void
-sqlite3_lessthan (sqlite3_context * ctx, int dummy, sqlite3_value ** values)
-{
-  uint64_t v1;
-  uint64_t v2;
-
-  v1 = (uint64_t) sqlite3_value_int64 (values[0]);
-  v2 = (uint64_t) sqlite3_value_int64 (values[1]);
-  sqlite3_result_int (ctx, v1 < v2);
-}
-
-
-/**
  * Initialize the database connections and associated
  * data structures (create tables and indices
  * as needed as well).
@@ -455,10 +506,13 @@ database_setup (struct Plugin *plugin)
   char *filename;
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (plugin->cfg, "peerstore-sqlite",
-                                               "FILENAME", &filename))
+      GNUNET_CONFIGURATION_get_value_filename (plugin->cfg,
+                                               "peerstore-sqlite",
+                                               "FILENAME",
+                                               &filename))
   {
-    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "peerstore-sqlite",
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                               "peerstore-sqlite",
                                "FILENAME");
     return GNUNET_SYSERR;
   }
@@ -474,60 +528,81 @@ database_setup (struct Plugin *plugin)
   /* filename should be UTF-8-encoded. If it isn't, it's a bug */
   plugin->fn = filename;
   /* Open database and precompile statements */
-  if (SQLITE_OK != sqlite3_open (plugin->fn, &plugin->dbh))
+  if (SQLITE_OK != sqlite3_open (plugin->fn,
+                                 &plugin->dbh))
   {
-    LOG (GNUNET_ERROR_TYPE_ERROR, _("Unable to initialize SQLite: %s.\n"),
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Unable to initialize SQLite: %s.\n"),
          sqlite3_errmsg (plugin->dbh));
     return GNUNET_SYSERR;
   }
-  sql_exec (plugin->dbh, "PRAGMA temp_store=MEMORY");
-  sql_exec (plugin->dbh, "PRAGMA synchronous=OFF");
-  sql_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF");
-  sql_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL");
-  sql_exec (plugin->dbh, "PRAGMA encoding=\"UTF-8\"");
-  sql_exec (plugin->dbh, "PRAGMA page_size=4096");
-  sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS);
+  sql_exec (plugin->dbh,
+            "PRAGMA temp_store=MEMORY");
+  sql_exec (plugin->dbh,
+            "PRAGMA synchronous=OFF");
+  sql_exec (plugin->dbh,
+            "PRAGMA legacy_file_format=OFF");
+  sql_exec (plugin->dbh,
+            "PRAGMA auto_vacuum=INCREMENTAL");
+  sql_exec (plugin->dbh,
+            "PRAGMA encoding=\"UTF-8\"");
+  sql_exec (plugin->dbh,
+            "PRAGMA page_size=4096");
+  sqlite3_busy_timeout (plugin->dbh,
+                        BUSY_TIMEOUT_MS);
   /* Create tables */
   sql_exec (plugin->dbh,
             "CREATE TABLE IF NOT EXISTS peerstoredata (\n"
-            "  sub_system TEXT NOT NULL,\n" "  peer_id BLOB NOT NULL,\n"
-            "  key TEXT NOT NULL,\n" "  value BLOB NULL,\n"
-            "  expiry sqlite3_uint64 NOT NULL" ");");
-  sqlite3_create_function (plugin->dbh, "UINT64_LT", 2, SQLITE_UTF8, NULL,
-                           &sqlite3_lessthan, NULL, NULL);
+            "  sub_system TEXT NOT NULL,\n"
+            "  peer_id BLOB NOT NULL,\n"
+            "  key TEXT NOT NULL,\n"
+            "  value BLOB NULL,\n"
+            "  expiry INT8 NOT NULL" ");");
   /* Create Indices */
   if (SQLITE_OK !=
       sqlite3_exec (plugin->dbh,
                     "CREATE INDEX IF NOT EXISTS peerstoredata_key_index ON peerstoredata (sub_system, peer_id, key)",
-                    NULL, NULL, NULL))
+                    NULL,
+                    NULL,
+                    NULL))
   {
-    LOG (GNUNET_ERROR_TYPE_ERROR, _("Unable to create indices: %s.\n"),
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Unable to create indices: %s.\n"),
          sqlite3_errmsg (plugin->dbh));
     return GNUNET_SYSERR;
   }
   /* Prepare statements */
 
   sql_prepare (plugin->dbh,
-               "INSERT INTO peerstoredata (sub_system, peer_id, key, value, expiry) VALUES (?,?,?,?,?);",
+               "INSERT INTO peerstoredata (sub_system, peer_id, key, value, expiry)"
+               " VALUES (?,?,?,?,?);",
                &plugin->insert_peerstoredata);
   sql_prepare (plugin->dbh,
-               "SELECT * FROM peerstoredata" " WHERE sub_system = ?",
+               "SELECT sub_system,peer_id,key,value,expiry FROM peerstoredata"
+               " WHERE sub_system = ?",
                &plugin->select_peerstoredata);
   sql_prepare (plugin->dbh,
-               "SELECT * FROM peerstoredata" " WHERE sub_system = ?"
-               " AND peer_id = ?", &plugin->select_peerstoredata_by_pid);
+               "SELECT sub_system,peer_id,key,value,expiry FROM peerstoredata"
+               " WHERE sub_system = ?"
+               " AND peer_id = ?",
+               &plugin->select_peerstoredata_by_pid);
   sql_prepare (plugin->dbh,
-               "SELECT * FROM peerstoredata" " WHERE sub_system = ?"
-               " AND key = ?", &plugin->select_peerstoredata_by_key);
+               "SELECT sub_system,peer_id,key,value,expiry FROM peerstoredata"
+               " WHERE sub_system = ?"
+               " AND key = ?",
+               &plugin->select_peerstoredata_by_key);
   sql_prepare (plugin->dbh,
-               "SELECT * FROM peerstoredata" " WHERE sub_system = ?"
+               "SELECT sub_system,peer_id,key,value,expiry FROM peerstoredata"
+               " WHERE sub_system = ?"
                " AND peer_id = ?" " AND key = ?",
                &plugin->select_peerstoredata_by_all);
   sql_prepare (plugin->dbh,
-               "DELETE FROM peerstoredata" " WHERE UINT64_LT(expiry, ?)",
+               "DELETE FROM peerstoredata"
+               " WHERE expiry < ?",
                &plugin->expire_peerstoredata);
   sql_prepare (plugin->dbh,
-               "DELETE FROM peerstoredata" " WHERE sub_system = ?"
+               "DELETE FROM peerstoredata"
+               " WHERE sub_system = ?"
                " AND peer_id = ?" " AND key = ?",
                &plugin->delete_peerstoredata);
   return GNUNET_OK;
@@ -545,15 +620,20 @@ database_shutdown (struct Plugin *plugin)
   int result;
   sqlite3_stmt *stmt;
 
-  while (NULL != (stmt = sqlite3_next_stmt (plugin->dbh, NULL)))
+  while (NULL != (stmt = sqlite3_next_stmt (plugin->dbh,
+                                            NULL)))
   {
     result = sqlite3_finalize (stmt);
     if (SQLITE_OK != result)
-      LOG (GNUNET_ERROR_TYPE_WARNING, "Failed to close statement %p: %d\n",
-           stmt, result);
+      LOG (GNUNET_ERROR_TYPE_WARNING,
+           "Failed to close statement %p: %d\n",
+           stmt,
+           result);
   }
   if (SQLITE_OK != sqlite3_close (plugin->dbh))
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite3_close");
   GNUNET_free_non_null (plugin->fn);
 }
 
@@ -573,7 +653,9 @@ libgnunet_plugin_peerstore_sqlite_init (void *cls)
 
   if (NULL != plugin.cfg)
     return NULL;                /* can only initialize once! */
-  memset (&plugin, 0, sizeof (struct Plugin));
+  memset (&plugin,
+          0,
+          sizeof (struct Plugin));
   plugin.cfg = cfg;
   if (GNUNET_OK != database_setup (&plugin))
   {
@@ -585,7 +667,8 @@ libgnunet_plugin_peerstore_sqlite_init (void *cls)
   api->store_record = &peerstore_sqlite_store_record;
   api->iterate_records = &peerstore_sqlite_iterate_records;
   api->expire_records = &peerstore_sqlite_expire_records;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Sqlite plugin is running\n");
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sqlite plugin is running\n");
   return api;
 }
 
@@ -605,7 +688,8 @@ libgnunet_plugin_peerstore_sqlite_done (void *cls)
   database_shutdown (plugin);
   plugin->cfg = NULL;
   GNUNET_free (api);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Sqlite plugin is finished\n");
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sqlite plugin is finished\n");
   return NULL;
 }
 
diff --git a/src/peerstore/test_peerstore_api_iterate.c b/src/peerstore/test_peerstore_api_iterate.c
index 83a6bf7b7..c607d9fb3 100644
--- a/src/peerstore/test_peerstore_api_iterate.c
+++ b/src/peerstore/test_peerstore_api_iterate.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C)
+     Copyright (C) 2013-2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -42,7 +42,8 @@ static int count = 0;
 
 
 static void
-iter3_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
+iter3_cb (void *cls,
+          const struct GNUNET_PEERSTORE_Record *record,
           const char *emsg)
 {
   if (NULL != emsg)
@@ -63,7 +64,8 @@ iter3_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
 
 
 static void
-iter2_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
+iter2_cb (void *cls,
+          const struct GNUNET_PEERSTORE_Record *record,
           const char *emsg)
 {
   if (NULL != emsg)
@@ -78,13 +80,19 @@ iter2_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
   }
   GNUNET_assert (count == 2);
   count = 0;
-  ic = GNUNET_PEERSTORE_iterate (h, ss, NULL, NULL, GNUNET_TIME_UNIT_FOREVER_REL,
-                            iter3_cb, NULL);
+  ic = GNUNET_PEERSTORE_iterate (h,
+                                 ss,
+                                 NULL,
+                                 NULL,
+                                 GNUNET_TIME_UNIT_FOREVER_REL,
+                                 &iter3_cb,
+                                 NULL);
 }
 
 
 static void
-iter1_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
+iter1_cb (void *cls,
+          const struct GNUNET_PEERSTORE_Record *record,
           const char *emsg)
 {
   if (NULL != emsg)
@@ -99,30 +107,61 @@ iter1_cb (void *cls, const struct GNUNET_PEERSTORE_Record *record,
   }
   GNUNET_assert (count == 1);
   count = 0;
-  ic = GNUNET_PEERSTORE_iterate (h, ss, &p1, NULL, GNUNET_TIME_UNIT_FOREVER_REL,
-                            iter2_cb, NULL);
+  ic = GNUNET_PEERSTORE_iterate (h,
+                                 ss,
+                                 &p1,
+                                 NULL,
+                                 GNUNET_TIME_UNIT_FOREVER_REL,
+                                 iter2_cb,
+                                 NULL);
 }
 
 
 static void
-run (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg,
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
      struct GNUNET_TESTING_Peer *peer)
 {
   h = GNUNET_PEERSTORE_connect (cfg);
   GNUNET_assert (NULL != h);
   memset (&p1, 1, sizeof (p1));
   memset (&p2, 2, sizeof (p2));
-  GNUNET_PEERSTORE_store (h, ss, &p1, k1, val, strlen (val) + 1,
+  GNUNET_PEERSTORE_store (h,
+                          ss,
+                          &p1,
+                          k1,
+                          val,
+                          strlen (val) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_REPLACE, NULL, NULL);
-  GNUNET_PEERSTORE_store (h, ss, &p1, k2, val, strlen (val) + 1,
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          NULL,
+                          NULL);
+  GNUNET_PEERSTORE_store (h,
+                          ss,
+                          &p1,
+                          k2,
+                          val,
+                          strlen (val) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_REPLACE, NULL, NULL);
-  GNUNET_PEERSTORE_store (h, ss, &p2, k3, val, strlen (val) + 1,
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          NULL,
+                          NULL);
+  GNUNET_PEERSTORE_store (h,
+                          ss,
+                          &p2,
+                          k3,
+                          val,
+                          strlen (val) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_REPLACE, NULL, NULL);
-  ic = GNUNET_PEERSTORE_iterate (h, ss, &p1, k1, GNUNET_TIME_UNIT_FOREVER_REL,
-                            iter1_cb, NULL);
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          NULL,
+                          NULL);
+  ic = GNUNET_PEERSTORE_iterate (h,
+                                 ss,
+                                 &p1,
+                                 k1,
+                                 GNUNET_TIME_UNIT_FOREVER_REL,
+                                 &iter1_cb, NULL);
 }
 
 
diff --git a/src/peerstore/test_peerstore_api_store.c b/src/peerstore/test_peerstore_api_store.c
index bfe1b5b55..978009dbb 100644
--- a/src/peerstore/test_peerstore_api_store.c
+++ b/src/peerstore/test_peerstore_api_store.c
@@ -39,6 +39,7 @@ static char *val3 = "test_peerstore_api_store_val3--";
 
 static int count = 0;
 
+
 static void
 test3_cont2 (void *cls,
              const struct GNUNET_PEERSTORE_Record *record,
@@ -49,7 +50,8 @@ test3_cont2 (void *cls,
   if (NULL != record)
   {
     GNUNET_assert ((strlen (val3) + 1) == record->value_size);
-    GNUNET_assert (0 == strcmp ((char *) val3, (char *) record->value));
+    GNUNET_assert (0 == strcmp ((char *) val3,
+                                (char *) record->value));
     count++;
     return;
   }
@@ -61,7 +63,8 @@ test3_cont2 (void *cls,
 
 
 static void
-test3_cont (void *cls, int success)
+test3_cont (void *cls,
+            int success)
 {
   if (GNUNET_YES != success)
     return;
@@ -71,7 +74,8 @@ test3_cont (void *cls, int success)
                             &pid,
                             key,
                             GNUNET_TIME_UNIT_SECONDS,
-                            &test3_cont2, NULL);
+                            &test3_cont2,
+                            NULL);
 }
 
 
@@ -81,9 +85,15 @@ test3_cont (void *cls, int success)
 static void
 test3 ()
 {
-  GNUNET_PEERSTORE_store (h, subsystem, &pid, key, val3, strlen (val3) + 1,
+  GNUNET_PEERSTORE_store (h,
+                          subsystem,
+                          &pid,
+                          key,
+                          val3,
+                          strlen (val3) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_REPLACE, &test3_cont,
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          &test3_cont,
                           NULL);
 }
 
@@ -130,9 +140,15 @@ test2_cont (void *cls, int success)
 void
 test2 ()
 {
-  GNUNET_PEERSTORE_store (h, subsystem, &pid, key, val2, strlen (val2) + 1,
+  GNUNET_PEERSTORE_store (h,
+                          subsystem,
+                          &pid,
+                          key,
+                          val2,
+                          strlen (val2) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_MULTIPLE, &test2_cont,
+                          GNUNET_PEERSTORE_STOREOPTION_MULTIPLE,
+                          &test2_cont,
                           NULL);
 }
 
@@ -163,8 +179,13 @@ test1_cont (void *cls, int success)
   if (GNUNET_YES != success)
     return;
   count = 0;
-  GNUNET_PEERSTORE_iterate (h, subsystem, &pid, key, GNUNET_TIME_UNIT_SECONDS,
-                            &test1_cont2, NULL);
+  GNUNET_PEERSTORE_iterate (h,
+                            subsystem,
+                            &pid,
+                            key,
+                            GNUNET_TIME_UNIT_SECONDS,
+                            &test1_cont2,
+                            NULL);
 }
 
 
@@ -174,9 +195,15 @@ test1_cont (void *cls, int success)
 static void
 test1 ()
 {
-  GNUNET_PEERSTORE_store (h, subsystem, &pid, key, val1, strlen (val1) + 1,
+  GNUNET_PEERSTORE_store (h,
+                          subsystem,
+                          &pid,
+                          key,
+                          val1,
+                          strlen (val1) + 1,
                           GNUNET_TIME_UNIT_FOREVER_ABS,
-                          GNUNET_PEERSTORE_STOREOPTION_REPLACE, &test1_cont,
+                          GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                          &test1_cont,
                           NULL);
 }
 
@@ -196,8 +223,10 @@ int
 main (int argc, char *argv[])
 {
   if (0 !=
-      GNUNET_TESTING_service_run ("test-gnunet-peerstore", "peerstore",
-                                  "test_peerstore_api_data.conf", &run, NULL))
+      GNUNET_TESTING_service_run ("test-gnunet-peerstore",
+                                  "peerstore",
+                                  "test_peerstore_api_data.conf",
+                                  &run, NULL))
     return 1;
   return ok;
 }
diff --git a/src/peerstore/test_plugin_peerstore.c b/src/peerstore/test_plugin_peerstore.c
index 179e32b52..62c06be8e 100644
--- a/src/peerstore/test_plugin_peerstore.c
+++ b/src/peerstore/test_plugin_peerstore.c
@@ -36,6 +36,11 @@ static int ok;
 static const char *plugin_name;
 
 
+static struct GNUNET_PEERSTORE_PluginFunctions *psp;
+
+static struct GNUNET_PeerIdentity p1;
+
+
 /**
  * Function called when the service shuts down.  Unloads our namestore
  * plugin.
@@ -47,8 +52,12 @@ unload_plugin (struct GNUNET_PEERSTORE_PluginFunctions *api)
 {
   char *libname;
 
-  GNUNET_asprintf (&libname, "libgnunet_plugin_peer_%s", plugin_name);
-  GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
+  GNUNET_asprintf (&libname,
+                   "libgnunet_plugin_peer_%s",
+                   plugin_name);
+  GNUNET_break (NULL ==
+                GNUNET_PLUGIN_unload (libname,
+                                      api));
   GNUNET_free (libname);
 }
 
@@ -65,12 +74,18 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
   struct GNUNET_PEERSTORE_PluginFunctions *ret;
   char *libname;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' peer plugin\n"),
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Loading `%s' peer plugin\n"),
               plugin_name);
-  GNUNET_asprintf (&libname, "libgnunet_plugin_peerstore_%s", plugin_name);
-  if (NULL == (ret = GNUNET_PLUGIN_load (libname, (void*) cfg)))
+  GNUNET_asprintf (&libname,
+                   "libgnunet_plugin_peerstore_%s",
+                   plugin_name);
+  if (NULL == (ret = GNUNET_PLUGIN_load (libname,
+                                         (void*) cfg)))
   {
-    FPRINTF (stderr, "Failed to load plugin `%s'!\n", plugin_name);
+    FPRINTF (stderr,
+             "Failed to load plugin `%s'!\n",
+             plugin_name);
     GNUNET_free (libname);
     return NULL;
   }
@@ -81,19 +96,28 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
 
 static void
 test_record (void *cls,
-                                                 const struct GNUNET_PEERSTORE_Record *record,
-                                                 const char *error)
+             const struct GNUNET_PEERSTORE_Record *record,
+             const char *error)
 {
-  struct GNUNET_PeerIdentity *id = cls;
-  char* testval = "test_val";
+  const struct GNUNET_PeerIdentity *id = cls;
+  const char* testval = "test_val";
 
   if (NULL == record)
+  {
+    unload_plugin (psp);
     return;
-
-  GNUNET_assert (0 == memcmp (record->peer, id, sizeof (struct GNUNET_PeerIdentity)));
-  GNUNET_assert (0 == strcmp ("subsys", record->sub_system));
-  GNUNET_assert (0 == strcmp ("key", record->key));
-  GNUNET_assert (0 == memcmp (testval, record->value, strlen (testval)));
+  }
+  GNUNET_assert (0 == memcmp (&record->peer,
+                              id,
+                              sizeof (struct GNUNET_PeerIdentity)));
+  GNUNET_assert (0 == strcmp ("subsys",
+                              record->sub_system));
+  GNUNET_assert (0 == strcmp ("key",
+                              record->key));
+  GNUNET_assert (0 == memcmp (testval,
+                              record->value,
+                              strlen (testval)));
+  ok = 0;
 }
 
 
@@ -101,38 +125,52 @@ static void
 get_record (struct GNUNET_PEERSTORE_PluginFunctions *psp,
             const struct GNUNET_PeerIdentity *identity)
 {
-  GNUNET_assert (GNUNET_OK == psp->iterate_records (psp->cls,
-                                            "subsys", identity, "key", &test_record, (void*)identity));
+  GNUNET_assert (GNUNET_OK ==
+                 psp->iterate_records (psp->cls,
+                                       "subsys",
+                                       identity,
+                                       "key",
+                                       &test_record,
+                                       (void*)identity));
 }
 
+
 static void
-store_cont (void *cls, int status)
+store_cont (void *cls,
+            int status)
 {
   GNUNET_assert (GNUNET_OK == status);
+  get_record (psp,
+              &p1);
 }
 
+
 static void
-put_record (struct GNUNET_PEERSTORE_PluginFunctions *psp, struct GNUNET_PeerIdentity *identity)
+put_record (struct GNUNET_PEERSTORE_PluginFunctions *psp,
+            const struct GNUNET_PeerIdentity *identity)
 {
-  GNUNET_assert (GNUNET_OK == psp->store_record (psp->cls,
-            "subsys",
-                                                identity,
-            "key", "test_value", strlen ("test_value"),
-            GNUNET_TIME_absolute_get (),
-            GNUNET_PEERSTORE_STOREOPTION_REPLACE,
-                                                &store_cont,
-            identity));
+  GNUNET_assert (GNUNET_OK ==
+                 psp->store_record (psp->cls,
+                                    "subsys",
+                                    identity,
+                                    "key",
+                                    "test_value",
+                                    strlen ("test_value"),
+                                    GNUNET_TIME_absolute_get (),
+                                    GNUNET_PEERSTORE_STOREOPTION_REPLACE,
+                                    &store_cont,
+                                    NULL));
 }
 
 
 static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
-  struct GNUNET_PEERSTORE_PluginFunctions *psp;
-  struct GNUNET_PeerIdentity p1;
 
-  ok = 0;
+  ok = 1;
   psp = load_plugin (cfg);
   if (NULL == psp)
   {
@@ -142,10 +180,8 @@ run (void *cls, char *const *args, const char *cfgfile,
     return;
   }
   memset (&p1, 1, sizeof (p1));
-  put_record (psp, &p1);
-  get_record (psp, &p1);
-
-  unload_plugin (psp);
+  put_record (psp,
+              &p1);
 }
 
 
@@ -163,19 +199,26 @@ main (int argc, char *argv[])
     GNUNET_GETOPT_OPTION_END
   };
 
-  //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite");
   GNUNET_log_setup ("test-plugin-peerstore",
                     "WARNING",
                     NULL);
   plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
-  GNUNET_snprintf (cfg_name, sizeof (cfg_name), "test_plugin_peerstore_%s.conf",
+  GNUNET_snprintf (cfg_name,
+                   sizeof (cfg_name),
+                   "test_plugin_peerstore_%s.conf",
                    plugin_name);
-  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv,
-                      "test-plugin-peerstore", "nohelp", options, &run, NULL);
+  GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1,
+                      xargv,
+                      "test-plugin-peerstore",
+                      "nohelp",
+                      options,
+                      &run,
+                      NULL);
   if (ok != 0)
-    FPRINTF (stderr, "Missed some testcases: %d\n", ok);
-  //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite");
+    FPRINTF (stderr,
+             "Missed some testcases: %d\n",
+             ok);
   return ok;
 }
 
-/* end of test_plugin_namestore.c */
+/* end of test_plugin_peerstore.c */
diff --git a/src/pq/pq.c b/src/pq/pq.c
index f4bab4e00..4e24e1ef8 100644
--- a/src/pq/pq.c
+++ b/src/pq/pq.c
@@ -85,9 +85,10 @@ GNUNET_PQ_exec_prepared (PGconn *db_conn,
       off += x->num_params;
     }
     GNUNET_assert (off == len);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Executing prepared SQL statement `%s'\n",
-                name);
+    GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                     "pq",
+                     "Executing prepared SQL statement `%s'\n",
+                     name);
     res = PQexecPrepared (db_conn,
                           name,
                           len,
diff --git a/src/psyc/.gitignore b/src/psyc/.gitignore
index e12b3210c..14a175367 100644
--- a/src/psyc/.gitignore
+++ b/src/psyc/.gitignore
@@ -1 +1,2 @@
 gnunet-service-psyc
+test_psyc
diff --git a/src/psyc/psyc_api.c b/src/psyc/psyc_api.c
index c6544df3a..c93d8b383 100644
--- a/src/psyc/psyc_api.c
+++ b/src/psyc/psyc_api.c
@@ -1309,7 +1309,7 @@ channel_history_replay (struct GNUNET_PSYC_Channel *chn,
 
   GNUNET_assert (NULL != method_prefix);
   uint16_t method_size = strnlen (method_prefix,
-                                  GNUNET_SERVER_MAX_MESSAGE_SIZE
+                                  GNUNET_MAX_MESSAGE_SIZE
                                   - sizeof (*req)) + 1;
   GNUNET_assert ('\0' == method_prefix[method_size - 1]);
 
@@ -1454,7 +1454,7 @@ channel_state_get (struct GNUNET_PSYC_Channel *chn,
   sr->op_id = GNUNET_OP_add (chn->op, op_recv_state_result, sr, NULL);
 
   GNUNET_assert (NULL != name);
-  size_t name_size = strnlen (name, GNUNET_SERVER_MAX_MESSAGE_SIZE
+  size_t name_size = strnlen (name, GNUNET_MAX_MESSAGE_SIZE
                               - sizeof (*req)) + 1;
   struct GNUNET_MQ_Envelope *
     env = GNUNET_MQ_msg_extra (req, name_size, type);
diff --git a/src/psycstore/.gitignore b/src/psycstore/.gitignore
index fc2e4cf8e..5ec783202 100644
--- a/src/psycstore/.gitignore
+++ b/src/psycstore/.gitignore
@@ -1 +1,5 @@
 gnunet-service-psycstore
+test_plugin_psycstore_mysql
+test_plugin_psycstore_sqlite
+test_plugin_psycstore_postgres
+test_psycstore
diff --git a/src/psycstore/gnunet-service-psycstore.c b/src/psycstore/gnunet-service-psycstore.c
index 10c92a878..1bee8da65 100644
--- a/src/psycstore/gnunet-service-psycstore.c
+++ b/src/psycstore/gnunet-service-psycstore.c
@@ -106,7 +106,7 @@ send_result_code (struct GNUNET_SERVICE_Client *client,
 
   if (NULL != err_msg)
     err_size = strnlen (err_msg,
-                        GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*res) - 1) + 1;
+                        GNUNET_MAX_MESSAGE_SIZE - sizeof (*res) - 1) + 1;
   struct GNUNET_MQ_Envelope *
     env = GNUNET_MQ_msg_extra (res, err_size,
                                GNUNET_MESSAGE_TYPE_PSYCSTORE_RESULT_CODE);
diff --git a/src/psycstore/psycstore_api.c b/src/psycstore/psycstore_api.c
index 40322662e..d79daa357 100644
--- a/src/psycstore/psycstore_api.c
+++ b/src/psycstore/psycstore_api.c
@@ -809,7 +809,7 @@ GNUNET_PSYCSTORE_message_get (struct GNUNET_PSYCSTORE_Handle *h,
   if (NULL == method_prefix)
     method_prefix = "";
   uint16_t method_size = strnlen (method_prefix,
-                                  GNUNET_SERVER_MAX_MESSAGE_SIZE
+                                  GNUNET_MAX_MESSAGE_SIZE
                                   - sizeof (*req)) + 1;
 
   struct GNUNET_MQ_Envelope *
@@ -875,7 +875,7 @@ GNUNET_PSYCSTORE_message_get_latest (struct GNUNET_PSYCSTORE_Handle *h,
   if (NULL == method_prefix)
     method_prefix = "";
   uint16_t method_size = strnlen (method_prefix,
-                                  GNUNET_SERVER_MAX_MESSAGE_SIZE
+                                  GNUNET_MAX_MESSAGE_SIZE
                                   - sizeof (*req)) + 1;
   GNUNET_assert ('\0' == method_prefix[method_size - 1]);
 
diff --git a/src/psycutil/.gitignore b/src/psycutil/.gitignore
new file mode 100644
index 000000000..03d8197fb
--- /dev/null
+++ b/src/psycutil/.gitignore
@@ -0,0 +1 @@
+test_psyc_env
diff --git a/src/pt/Makefile.am b/src/pt/Makefile.am
index 7ea8257d5..e36630ae4 100644
--- a/src/pt/Makefile.am
+++ b/src/pt/Makefile.am
@@ -25,7 +25,7 @@ gnunet_daemon_pt_SOURCES = \
  gnunet-daemon-pt.c
 gnunet_daemon_pt_LDADD = \
   $(top_builddir)/src/vpn/libgnunetvpn.la \
-  $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la \
   $(top_builddir)/src/dht/libgnunetdht.la \
   $(top_builddir)/src/dns/libgnunetdns.la \
   $(top_builddir)/src/dns/libgnunetdnsparser.la \
diff --git a/src/pt/gnunet-daemon-pt.c b/src/pt/gnunet-daemon-pt.c
index 54556cc52..295082c0e 100644
--- a/src/pt/gnunet-daemon-pt.c
+++ b/src/pt/gnunet-daemon-pt.c
@@ -1015,7 +1015,7 @@ cadet_channel_end_cb (void *cls,
     GNUNET_CONTAINER_DLL_insert (alt->receive_queue_head,
                                  alt->receive_queue_tail,
                                  rc);
-    GNUNET_MQ_send (GNUNET_CADET_get_mq (exit->cadet_channel),
+    GNUNET_MQ_send (GNUNET_CADET_get_mq (alt->cadet_channel),
                     GNUNET_MQ_env_copy (rc->env));
   }
   try_open_exit ();
@@ -1084,7 +1084,7 @@ try_open_exit ()
 
         /* move to the head of the DLL */
         pos->cadet_channel
-          = GNUNET_CADET_channel_creatE (cadet_handle,
+          = GNUNET_CADET_channel_create (cadet_handle,
                                          pos,
                                          &pos->peer,
                                          &port,
@@ -1246,7 +1246,7 @@ run (void *cls, char *const *args GNUNET_UNUSED,
       GNUNET_SCHEDULER_shutdown ();
       return;
     }
-    cadet_handle = GNUNET_CADET_connecT (cfg);
+    cadet_handle = GNUNET_CADET_connect (cfg);
     if (NULL == cadet_handle)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
diff --git a/src/regex/gnunet-regex-profiler.c b/src/regex/gnunet-regex-profiler.c
index dfbcd388a..71f3580f8 100644
--- a/src/regex/gnunet-regex-profiler.c
+++ b/src/regex/gnunet-regex-profiler.c
@@ -1477,22 +1477,39 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'o', "output-file", "FILENAME",
-     gettext_noop ("name of the file for writing statistics"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &data_filename},
-    {'t', "matching-timeout", "TIMEOUT",
-      gettext_noop ("wait TIMEOUT before ending the experiment"),
-      GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &search_timeout_time},
-    {'p', "policy-dir", "DIRECTORY",
-      gettext_noop ("directory with policy files"),
-      GNUNET_YES, &GNUNET_GETOPT_set_filename, &policy_dir},
-    {'s', "strings-file", "FILENAME",
-      gettext_noop ("name of file with input strings"),
-      GNUNET_YES, &GNUNET_GETOPT_set_filename, &strings_file},
-    {'H', "hosts-file", "FILENAME",
-      gettext_noop ("name of file with hosts' names"),
-      GNUNET_YES, &GNUNET_GETOPT_set_filename, &hosts_file},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_FILENAME ('o',
+                                   "output-file",
+                                   "FILENAME",
+                                   gettext_noop ("name of the file for writing statistics"),
+                                   &data_filename),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('t',
+                                            "matching-timeout",
+                                            "TIMEOUT",
+                                            gettext_noop ("wait TIMEOUT before ending the experiment"),
+                                            &search_timeout_time), 
+
+    GNUNET_GETOPT_OPTION_FILENAME ('p',
+                                   "policy-dir",
+                                   "DIRECTORY",
+                                   gettext_noop ("directory with policy files"),
+                                   &policy_dir),
+
+
+    GNUNET_GETOPT_OPTION_FILENAME ('s',
+                                   "strings-file",
+                                   "FILENAME",
+                                   gettext_noop ("name of file with input strings"),
+                                   &strings_file),
+
+    GNUNET_GETOPT_OPTION_FILENAME ('H',
+                                   "hosts-file",
+                                   "FILENAME",
+                                   gettext_noop ("name of file with hosts' names"),
+                                   &hosts_file),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/regex/gnunet-regex-simulation-profiler.c b/src/regex/gnunet-regex-simulation-profiler.c
index 025a70316..422e45565 100644
--- a/src/regex/gnunet-regex-simulation-profiler.c
+++ b/src/regex/gnunet-regex-simulation-profiler.c
@@ -691,13 +691,20 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'t', "table", "TABLENAME",
-     gettext_noop ("name of the table to write DFAs"),
-     1, &GNUNET_GETOPT_set_string, &table_name},
-    {'p', "max-path-compression", "MAX_PATH_COMPRESSION",
-     gettext_noop ("maximum path compression length"),
-     1, &GNUNET_GETOPT_set_uint, &max_path_compression},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_STRING ('t',
+                                 "table",
+                                 "TABLENAME",
+                                 gettext_noop ("name of the table to write DFAs"),
+                                 &table_name),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('p',
+                                   "max-path-compression",
+                                   "MAX_PATH_COMPRESSION",
+                                   gettext_noop ("maximum path compression length"),
+                                   &max_path_compression),
+
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/regex/gnunet-service-regex.c b/src/regex/gnunet-service-regex.c
index 294670be6..9a40a5264 100644
--- a/src/regex/gnunet-service-regex.c
+++ b/src/regex/gnunet-service-regex.c
@@ -37,19 +37,14 @@ struct ClientEntry
 {
 
   /**
-   * Kept in DLL.
+   * Queue for transmissions to @e client.
    */
-  struct ClientEntry *next;
-
-  /**
-   * Kept in DLL.
-   */
-  struct ClientEntry *prev;
+  struct GNUNET_MQ_Handle *mq;
 
   /**
    * Handle identifying the client.
    */
-  struct GNUNET_SERVER_Client *client;
+  struct GNUNET_SERVICE_Client *client;
 
   /**
    * Search handle (if this client is searching).
@@ -69,7 +64,7 @@ struct ClientEntry
   /**
    * Task for re-announcing.
    */
-  struct GNUNET_SCHEDULER_Task * refresh_task;
+  struct GNUNET_SCHEDULER_Task *refresh_task;
 
 };
 
@@ -85,70 +80,12 @@ static struct GNUNET_DHT_Handle *dht;
 static struct GNUNET_STATISTICS_Handle *stats;
 
 /**
- * Head of list of clients.
- */
-static struct ClientEntry *client_head;
-
-/**
- * End of list of clients.
- */
-static struct ClientEntry *client_tail;
-
-/**
- * Our notification context, used to send back results to the client.
- */
-static struct GNUNET_SERVER_NotificationContext *nc;
-
-/**
  * Private key for this peer.
  */
 static struct GNUNET_CRYPTO_EddsaPrivateKey *my_private_key;
 
 
 /**
- * A client disconnected.  Remove all of its data structure entries.
- *
- * @param cls closure, NULL
- * @param client identification of the client
- */
-static void
-handle_client_disconnect (void *cls,
-                          struct GNUNET_SERVER_Client *client)
-{
-  struct ClientEntry *ce;
-  struct ClientEntry *nx;
-
-  nx = client_head;
-  for (ce = nx; NULL != ce; ce = nx)
-  {
-    nx = ce->next;
-    if (ce->client == client)
-    {
-      if (NULL != ce->refresh_task)
-      {
-        GNUNET_SCHEDULER_cancel (ce->refresh_task);
-        ce->refresh_task = NULL;
-      }
-      if (NULL != ce->ah)
-      {
-        REGEX_INTERNAL_announce_cancel (ce->ah);
-        ce->ah = NULL;
-      }
-      if (NULL != ce->sh)
-      {
-        REGEX_INTERNAL_search_cancel (ce->sh);
-        ce->sh = NULL;
-      }
-      GNUNET_CONTAINER_DLL_remove (client_head,
-                                   client_tail,
-                                   ce);
-      GNUNET_free (ce);
-    }
-  }
-}
-
-
-/**
  * Task run during shutdown.
  *
  * @param cls unused
@@ -156,17 +93,11 @@ handle_client_disconnect (void *cls,
 static void
 cleanup_task (void *cls)
 {
-  struct ClientEntry *ce;
-
-  while (NULL != (ce = client_head))
-    handle_client_disconnect (NULL,
-                              ce->client);
   GNUNET_DHT_disconnect (dht);
   dht = NULL;
-  GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
+  GNUNET_STATISTICS_destroy (stats,
+                             GNUNET_NO);
   stats = NULL;
-  GNUNET_SERVER_notification_context_destroy (nc);
-  nc = NULL;
   GNUNET_free (my_private_key);
   my_private_key = NULL;
 }
@@ -191,35 +122,51 @@ reannounce (void *cls)
 
 
 /**
- * Handle ANNOUNCE message.
+ * Check ANNOUNCE message.
  *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
+ * @param cls identification of the client
+ * @param am the actual message
+ * @return #GNUNET_OK if @am is well-formed
  */
-static void
-handle_announce (void *cls,
-                 struct GNUNET_SERVER_Client *client,
-                 const struct GNUNET_MessageHeader *message)
+static int
+check_announce (void *cls,
+                const struct AnnounceMessage *am)
 {
-  const struct AnnounceMessage *am;
+  struct ClientEntry *ce = cls;
   const char *regex;
-  struct ClientEntry *ce;
   uint16_t size;
 
-  size = ntohs (message->size);
-  am = (const struct AnnounceMessage *) message;
+  size = ntohs (am->header.size) - sizeof (*am);
   regex = (const char *) &am[1];
-  if ( (size <= sizeof (struct AnnounceMessage)) ||
-       ('\0' != regex[size - sizeof (struct AnnounceMessage) - 1]) )
+  if ('\0' != regex[size - 1])
   {
     GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
+    return GNUNET_SYSERR;
+  }
+  if (NULL != ce->ah)
+  {
+    /* only one announcement per client allowed */
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
 
-  ce = GNUNET_new (struct ClientEntry);
-  ce->client = client;
+
+/**
+ * Handle ANNOUNCE message.
+ *
+ * @param cls identification of the client
+ * @param am the actual message
+ */
+static void
+handle_announce (void *cls,
+                 const struct AnnounceMessage *am)
+{
+  struct ClientEntry *ce = cls;
+  const char *regex;
+
+  regex = (const char *) &am[1];
   ce->frequency = GNUNET_TIME_relative_ntoh (am->refresh_delay);
   ce->refresh_task = GNUNET_SCHEDULER_add_delayed (ce->frequency,
                                                    &reannounce,
@@ -238,14 +185,11 @@ handle_announce (void *cls,
   {
     GNUNET_break (0);
     GNUNET_SCHEDULER_cancel (ce->refresh_task);
-    GNUNET_free (ce);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    ce->refresh_task = NULL;
+    GNUNET_SERVICE_client_drop (ce->client);
     return;
   }
-  GNUNET_CONTAINER_DLL_insert (client_head,
-                               client_tail,
-                               ce);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  GNUNET_SERVICE_client_continue (ce->client);
 }
 
 
@@ -268,6 +212,7 @@ handle_search_result (void *cls,
                       unsigned int put_path_length)
 {
   struct ClientEntry *ce = cls;
+  struct GNUNET_MQ_Envelope *env;
   struct ResultMessage *result;
   struct GNUNET_PeerIdentity *gp;
   uint16_t size;
@@ -275,64 +220,78 @@ handle_search_result (void *cls,
   if ( (get_path_length >= 65536) ||
        (put_path_length >= 65536) ||
        ( (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity))
-       + sizeof (struct ResultMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+       + sizeof (struct ResultMessage) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
   }
-  size = (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity) + sizeof (struct ResultMessage);
-  result = GNUNET_malloc (size);
-  result->header.size = htons (size);
-  result->header.type = htons (GNUNET_MESSAGE_TYPE_REGEX_RESULT);
+  size = (get_path_length + put_path_length) * sizeof (struct GNUNET_PeerIdentity);
+  env = GNUNET_MQ_msg_extra (result,
+                             size,
+                             GNUNET_MESSAGE_TYPE_REGEX_RESULT);
   result->get_path_length = htons ((uint16_t) get_path_length);
   result->put_path_length = htons ((uint16_t) put_path_length);
   result->id = *id;
   gp = &result->id;
   GNUNET_memcpy (&gp[1],
-          get_path,
-          get_path_length * sizeof (struct GNUNET_PeerIdentity));
+                 get_path,
+                 get_path_length * sizeof (struct GNUNET_PeerIdentity));
   GNUNET_memcpy (&gp[1 + get_path_length],
-          put_path,
-          put_path_length * sizeof (struct GNUNET_PeerIdentity));
-  GNUNET_SERVER_notification_context_unicast (nc,
-                                              ce->client,
-                                              &result->header, GNUNET_NO);
-  GNUNET_free (result);
+                 put_path,
+                 put_path_length * sizeof (struct GNUNET_PeerIdentity));
+  GNUNET_MQ_send (ce->mq,
+                  env);
 }
 
 
 /**
- * Handle SEARCH message.
+ * Check SEARCH message.
  *
- * @param cls closure
- * @param client identification of the client
+ * @param cls identification of the client
  * @param message the actual message
  */
-static void
-handle_search (void *cls,
-               struct GNUNET_SERVER_Client *client,
-               const struct GNUNET_MessageHeader *message)
+static int
+check_search (void *cls,
+              const struct RegexSearchMessage *sm)
 {
-  const struct RegexSearchMessage *sm;
+  struct ClientEntry *ce = cls;
   const char *string;
-  struct ClientEntry *ce;
   uint16_t size;
 
-  size = ntohs (message->size);
-  sm = (const struct RegexSearchMessage *) message;
+  size = ntohs (sm->header.size) - sizeof (*sm);
   string = (const char *) &sm[1];
-  if ( (size <= sizeof (struct RegexSearchMessage)) ||
-       ('\0' != string[size - sizeof (struct RegexSearchMessage) - 1]) )
+  if ('\0' != string[size - 1])
   {
     GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
+    return GNUNET_SYSERR;
   }
+  if (NULL != ce->sh)
+  {
+    /* only one search allowed per client */
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle SEARCH message.
+ *
+ * @param cls identification of the client
+ * @param message the actual message
+ */
+static void
+handle_search (void *cls,
+               const struct RegexSearchMessage *sm)
+{
+  struct ClientEntry *ce = cls;
+  const char *string;
+
+  string = (const char *) &sm[1];
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Starting to search for `%s'\n",
               string);
-  ce = GNUNET_new (struct ClientEntry);
-  ce->client = client;
   ce->sh = REGEX_INTERNAL_search (dht,
                                   string,
                                   &handle_search_result,
@@ -341,15 +300,10 @@ handle_search (void *cls,
   if (NULL == ce->sh)
   {
     GNUNET_break (0);
-    GNUNET_free (ce);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    GNUNET_SERVICE_client_drop (ce->client);
     return;
   }
-  GNUNET_CONTAINER_DLL_insert (client_head,
-                               client_tail,
-                               ce);
-  GNUNET_SERVER_notification_context_add (nc, client);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  GNUNET_SERVICE_client_continue (ce->client);
 }
 
 
@@ -357,19 +311,14 @@ handle_search (void *cls,
  * Process regex requests.
  *
  * @param cls closure
- * @param server the initialized server
  * @param cfg configuration to use
+ * @param service the initialized service
  */
 static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
-     const struct GNUNET_CONFIGURATION_Handle *cfg)
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *cfg,
+     struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_SERVER_MessageHandler handlers[] = {
-    {&handle_announce, NULL, GNUNET_MESSAGE_TYPE_REGEX_ANNOUNCE, 0},
-    {&handle_search, NULL, GNUNET_MESSAGE_TYPE_REGEX_SEARCH, 0},
-    {NULL, NULL, 0, 0}
-  };
-
   my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (cfg);
   if (NULL == my_private_key)
   {
@@ -386,28 +335,84 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
   }
   GNUNET_SCHEDULER_add_shutdown (&cleanup_task,
                                  NULL);
-  nc = GNUNET_SERVER_notification_context_create (server, 1);
   stats = GNUNET_STATISTICS_create ("regex", cfg);
-  GNUNET_SERVER_add_handlers (server, handlers);
-  GNUNET_SERVER_disconnect_notify (server,
-                                   &handle_client_disconnect,
-                                   NULL);
 }
 
 
 /**
- * The main function for the regex service.
+ * Callback called when a client connects to the service.
  *
- * @param argc number of arguments from the command line
- * @param argv command line arguments
- * @return 0 ok, 1 on error
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
  */
-int
-main (int argc, char *const *argv)
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
 {
-  return (GNUNET_OK ==
-          GNUNET_SERVICE_run (argc, argv, "regex",
-                              GNUNET_SERVICE_OPTION_NONE, &run, NULL)) ? 0 : 1;
+  struct ClientEntry *ce;
+
+  ce = GNUNET_new (struct ClientEntry);
+  ce->client = c;
+  ce->mq = mq;
+  return ce;
 }
 
+
+/**
+ * Callback called when a client disconnected from the service
+ *
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls should be equal to @a c
+ */
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *c,
+                      void *internal_cls)
+{
+  struct ClientEntry *ce = internal_cls;
+
+  if (NULL != ce->refresh_task)
+  {
+    GNUNET_SCHEDULER_cancel (ce->refresh_task);
+    ce->refresh_task = NULL;
+  }
+  if (NULL != ce->ah)
+  {
+    REGEX_INTERNAL_announce_cancel (ce->ah);
+    ce->ah = NULL;
+  }
+  if (NULL != ce->sh)
+  {
+    REGEX_INTERNAL_search_cancel (ce->sh);
+    ce->sh = NULL;
+  }
+  GNUNET_free (ce);
+}
+
+
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+("regex",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_var_size (announce,
+                        GNUNET_MESSAGE_TYPE_REGEX_ANNOUNCE,
+                        struct AnnounceMessage,
+                        NULL),
+ GNUNET_MQ_hd_var_size (search,
+                        GNUNET_MESSAGE_TYPE_REGEX_SEARCH,
+                        struct RegexSearchMessage,
+                        NULL),
+ GNUNET_MQ_handler_end ());
+
+
 /* end of gnunet-service-regex.c */
diff --git a/src/regex/regex_api_announce.c b/src/regex/regex_api_announce.c
index 70bf34bc8..8e018f26c 100644
--- a/src/regex/regex_api_announce.c
+++ b/src/regex/regex_api_announce.c
@@ -146,7 +146,7 @@ GNUNET_REGEX_announce (const struct GNUNET_CONFIGURATION_Handle *cfg,
   size_t slen;
 
   slen = strlen (regex) + 1;
-  if (slen + sizeof (struct AnnounceMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (slen + sizeof (struct AnnounceMessage) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _("Regex `%s' is too long!\n"),
diff --git a/src/regex/regex_api_search.c b/src/regex/regex_api_search.c
index b7a015f87..a5480ac7a 100644
--- a/src/regex/regex_api_search.c
+++ b/src/regex/regex_api_search.c
@@ -206,7 +206,7 @@ GNUNET_REGEX_search (const struct GNUNET_CONFIGURATION_Handle *cfg,
   struct GNUNET_REGEX_Search *s;
   size_t slen = strlen (string) + 1;
 
-  if (slen + sizeof (struct RegexSearchMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (slen + sizeof (struct RegexSearchMessage) >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _("Search string `%s' is too long!\n"),
diff --git a/src/rest/gnunet-rest-server.c b/src/rest/gnunet-rest-server.c
index b115deb20..0b6c18267 100644
--- a/src/rest/gnunet-rest-server.c
+++ b/src/rest/gnunet-rest-server.c
@@ -66,7 +66,7 @@ static struct GNUNET_SCHEDULER_Task *httpd_task;
 /**
  * The port the service is running on (default 7776)
  */
-static unsigned long port = GNUNET_REST_SERVICE_PORT;
+static unsigned long long port = GNUNET_REST_SERVICE_PORT;
 
 /**
  * The listen socket of the service for IPv4
@@ -748,7 +748,7 @@ run (void *cls,
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Service listens on port %lu\n",
+              "Service listens on port %llu\n",
               port);
   httpd = MHD_start_daemon (MHD_USE_DEBUG | MHD_USE_NO_LISTEN_SOCKET,
                             0,
@@ -783,10 +783,12 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'p', "port", NULL,
-      gettext_noop ("listen on specified port (default: 7776)"), 1,
-      &GNUNET_GETOPT_set_ulong, &port},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ULONG ('p',
+                                    "port",
+                                    "PORT",
+                                    gettext_noop ("listen on specified port (default: 7776)"),
+                                    &port),
     GNUNET_GETOPT_OPTION_END
   };
   static const char* err_page =
diff --git a/src/revocation/gnunet-revocation.c b/src/revocation/gnunet-revocation.c
index 133468789..7b40c83d7 100644
--- a/src/revocation/gnunet-revocation.c
+++ b/src/revocation/gnunet-revocation.c
@@ -527,19 +527,31 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'f', "filename", "NAME",
-     gettext_noop ("use NAME for the name of the revocation file"),
-     1, &GNUNET_GETOPT_set_string, &filename},
-    {'R', "revoke", "NAME",
-     gettext_noop ("revoke the private key associated for the the private key associated with the ego NAME "),
-     1, &GNUNET_GETOPT_set_string, &revoke_ego},
-    {'p', "perform", NULL,
-     gettext_noop ("actually perform revocation, otherwise we just do the precomputation"),
-     0, &GNUNET_GETOPT_set_one, &perform},
-    {'t', "test", "KEY",
-     gettext_noop ("test if the public key KEY has been revoked"),
-     1, &GNUNET_GETOPT_set_string, &test_ego},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_STRING ('f',
+                                 "filename",
+                                 "NAME",
+                                 gettext_noop ("use NAME for the name of the revocation file"),
+                                 &filename),
+
+    GNUNET_GETOPT_OPTION_STRING ('R',
+                                 "revoke",
+                                 "NAME",
+                                 gettext_noop ("revoke the private key associated for the the private key associated with the ego NAME "),
+                                 &revoke_ego), 
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('p',
+                                  "perform",
+                                  gettext_noop ("actually perform revocation, otherwise we just do the precomputation"),
+                                  &perform),
+
+    GNUNET_GETOPT_OPTION_STRING ('t',
+                                 "test",
+                                 "KEY",
+                                 gettext_noop ("test if the public key KEY has been revoked"),
+                                 &test_ego), 
+
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/revocation/test_revocation.c b/src/revocation/test_revocation.c
index d3bbb879a..8d5593694 100644
--- a/src/revocation/test_revocation.c
+++ b/src/revocation/test_revocation.c
@@ -104,8 +104,8 @@ revocation_remote_cb (void *cls,
 
   if (GNUNET_NO == is_valid)
   {
-    fprintf (stderr,
-             "Local revocation successful\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Local revocation successful\n");
     ok = 0;
     GNUNET_SCHEDULER_shutdown ();
     return;
@@ -118,8 +118,8 @@ revocation_remote_cb (void *cls,
                                   NULL);
     return;
   }
-  fprintf (stderr,
-           "Flooding of revocation failed\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+              "Flooding of revocation failed\n");
   ok = 2;
   GNUNET_SCHEDULER_shutdown ();
 }
@@ -141,8 +141,8 @@ revocation_cb (void *cls,
   testpeers[1].revok_handle = NULL;
   if (GNUNET_NO == is_valid)
   {
-    fprintf (stderr,
-             "Revocation successful\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Revocation successful\n");
     check_revocation (NULL);
   }
 }
@@ -386,8 +386,8 @@ test_connection (void *cls,
    /* We are generating a CLIQUE */
   if (NUM_TEST_PEERS * (NUM_TEST_PEERS -1) == links_succeeded)
   {
-    fprintf (stderr,
-             "Testbed connected peers, initializing test\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Testbed connected peers, initializing test\n");
     for (c = 0; c < num_peers; c++)
     {
       testpeers[c].p = peers[c];
@@ -403,8 +403,8 @@ test_connection (void *cls,
   }
   else
   {
-    fprintf (stderr,
-             "Testbed failed to connect peers\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Testbed failed to connect peers\n");
     ok = 5;
     GNUNET_SCHEDULER_shutdown ();
     return;
diff --git a/src/rps/Makefile.am b/src/rps/Makefile.am
index de7f853c1..e6c8cd929 100644
--- a/src/rps/Makefile.am
+++ b/src/rps/Makefile.am
@@ -100,22 +100,28 @@ AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PAT
 TESTS = $(check_PROGRAMS)
 endif
 
-test_service_rps_view_SOURCES = gnunet-service-rps_view.h gnunet-service-rps_view.c \
-                                                                                                                                test_service_rps_view.c
+test_service_rps_view_SOURCES = \
+  gnunet-service-rps_view.h gnunet-service-rps_view.c \
+  test_service_rps_view.c
 test_service_rps_view_LDADD = $(top_builddir)/src/util/libgnunetutil.la
 
-test_service_rps_peers_SOURCES = gnunet-service-rps_peers.h gnunet-service-rps_peers.c \
-                                                                                                                                 test_service_rps_peers.c
-test_service_rps_peers_LDADD = $(top_builddir)/src/util/libgnunetutil.la \
-                                                                                                                         $(top_builddir)/src/cadet/libgnunetcadet.la
-
-test_service_rps_custommap_SOURCES = gnunet-service-rps_custommap.h gnunet-service-rps_custommap.c \
-                                                                                                                                 test_service_rps_custommap.c
-test_service_rps_custommap_LDADD = $(top_builddir)/src/util/libgnunetutil.la
-
-test_service_rps_sampler_elem_SOURCES = gnunet-service-rps_sampler_elem.h gnunet-service-rps_sampler_elem.c \
-                                                                                                                                                                rps-test_util.h rps-test_util.c \
-                                                                                                                                        test_service_rps_sampler_elem.c
+test_service_rps_peers_SOURCES = \
+  gnunet-service-rps_peers.h gnunet-service-rps_peers.c \
+  test_service_rps_peers.c
+test_service_rps_peers_LDADD = \
+  $(top_builddir)/src/util/libgnunetutil.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la
+
+test_service_rps_custommap_SOURCES = \
+  gnunet-service-rps_custommap.h gnunet-service-rps_custommap.c \
+  test_service_rps_custommap.c
+test_service_rps_custommap_LDADD = \
+  $(top_builddir)/src/util/libgnunetutil.la
+
+test_service_rps_sampler_elem_SOURCES = \
+  gnunet-service-rps_sampler_elem.h gnunet-service-rps_sampler_elem.c \
+  rps-test_util.h rps-test_util.c \
+  test_service_rps_sampler_elem.c
 test_service_rps_sampler_elem_LDADD = $(top_builddir)/src/util/libgnunetutil.la
 
 test_rps_malicious_1_SOURCES = $(rps_test_src)
diff --git a/src/rps/gnunet-rps.c b/src/rps/gnunet-rps.c
index 3dbb8053e..e36e547fa 100644
--- a/src/rps/gnunet-rps.c
+++ b/src/rps/gnunet-rps.c
@@ -43,43 +43,7 @@ static struct GNUNET_RPS_Request_Handle *req_handle;
 /**
  * PeerID (Option --seed)
  */
-static struct GNUNET_PeerIdentity *peer_id;
-
-
-/**
- * Set an option of type 'struct GNUNET_PeerIdentity *' from the command line.
- * A pointer to this function should be passed as part of the
- * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
- * of this type.  It should be followed by a pointer to a value of
- * type 'struct GNUNET_PeerIdentity *', which will be allocated with the requested string.
- *
- * @param ctx command line processing context
- * @param scls additional closure (will point to the 'char *',
- *             which will be allocated)
- * @param option name of the option
- * @param value actual value of the option (a PeerID)
- * @return #GNUNET_OK
- */
-static int
-GNUNET_GETOPT_set_peerid (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                          void *scls, const char *option, const char *value)
-{
-  struct GNUNET_PeerIdentity **val = (struct GNUNET_PeerIdentity **) scls;
-
-  GNUNET_assert (NULL != value);
-  GNUNET_free_non_null (*val);
-  /* Not quite sure whether that is a sane way */
-  *val = GNUNET_new (struct GNUNET_PeerIdentity);
-  if (GNUNET_OK !=
-      GNUNET_CRYPTO_eddsa_public_key_from_string (value,
-                                                  strlen (value),
-                                                  &((*val)->public_key)))
-  {
-    FPRINTF (stderr, "Invalid peer ID %s\n", value);
-    return GNUNET_SYSERR;
-  }
-  return GNUNET_OK;
-}
+static struct GNUNET_PeerIdentity peer_id;
 
 
 /**
@@ -139,10 +103,13 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   static uint64_t num_peers;
+  static struct GNUNET_PeerIdentity zero_pid;
 
   rps_handle = GNUNET_RPS_connect (cfg);
 
-  if (NULL == peer_id)
+  if (0 == memcmp (&zero_pid,
+                   &peer_id,
+                   sizeof (peer_id)))
   { /* Request n PeerIDs */
     /* If number was specified use it, else request single peer. */
     num_peers = (NULL == args[0]) ? 1 : atoi (args[0]);
@@ -153,8 +120,8 @@ run (void *cls,
   }
   else
   { /* Seed PeerID */
-    GNUNET_RPS_seed_ids (rps_handle, 1, peer_id);
-    FPRINTF (stdout, "Seeded PeerID %s\n", GNUNET_i2s_full (peer_id));
+    GNUNET_RPS_seed_ids (rps_handle, 1, &peer_id);
+    FPRINTF (stdout, "Seeded PeerID %s\n", GNUNET_i2s_full (&peer_id));
     ret = 0;
     GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
   }
@@ -172,10 +139,12 @@ main (int argc, char *const *argv)
 {
   const char helpstr[] =
     "Get random GNUnet peers. If none is specified a single is requested.";
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'s', "seed", "PEER_ID",
-      gettext_noop ("Seed a PeerID"),
-      GNUNET_YES, &GNUNET_GETOPT_set_peerid, &peer_id},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_BASE32_AUTO ('s',
+                                          "seed",
+                                          "PEER_ID",
+                                          gettext_noop ("Seed a PeerID"),
+                                          &peer_id),
     GNUNET_GETOPT_OPTION_END
   };
   return (GNUNET_OK ==
diff --git a/src/rps/gnunet-service-rps.c b/src/rps/gnunet-service-rps.c
index adcfe7d02..0a4543b30 100644
--- a/src/rps/gnunet-service-rps.c
+++ b/src/rps/gnunet-service-rps.c
@@ -217,6 +217,11 @@ static struct GNUNET_NSE_Handle *nse;
 static struct GNUNET_CADET_Handle *cadet_handle;
 
 /**
+ * @brief Port to communicate to other peers.
+ */
+static struct GNUNET_CADET_Port *cadet_port;
+
+/**
  * Handler to PEERINFO.
  */
 static struct GNUNET_PEERINFO_Handle *peerinfo_handle;
@@ -838,14 +843,10 @@ clean_peer (const struct GNUNET_PeerIdentity *peer)
  */
 static void
 cleanup_destroyed_channel (void *cls,
-                           const struct GNUNET_CADET_Channel *channel,
-                           void *channel_ctx)
+                           const struct GNUNET_CADET_Channel *channel)
 {
-  struct GNUNET_PeerIdentity *peer;
-
-  peer = (struct GNUNET_PeerIdentity *) GNUNET_CADET_channel_get_info (
-      (struct GNUNET_CADET_Channel *) channel, GNUNET_CADET_OPTION_PEER);
-       // FIXME wait for cadet to change this function
+  struct GNUNET_PeerIdentity *peer = cls;
+  uint32_t *channel_flag;
 
   if (GNUNET_NO == Peers_check_peer_known (peer))
   { /* We don't know a context to that peer */
@@ -858,7 +859,7 @@ cleanup_destroyed_channel (void *cls,
   if (GNUNET_YES == Peers_check_peer_flag (peer, Peers_TO_DESTROY))
   { /* We are in the middle of removing that peer from our knowledge. In this
        case simply make sure that the channels are cleaned. */
-    Peers_cleanup_destroyed_channel (cls, channel, channel_ctx);
+    Peers_cleanup_destroyed_channel (cls, channel);
     to_file (file_name_view_log,
              "-%s\t(cleanup channel, ourself)",
              GNUNET_i2s_full (peer));
@@ -872,16 +873,17 @@ cleanup_destroyed_channel (void *cls,
      *  - ourselves  -> cleaning send channel -> clean context
      *  - other peer -> peer probably went down -> remove
      */
-    if (GNUNET_YES == Peers_check_channel_flag (channel_ctx, Peers_CHANNEL_CLEAN))
+    channel_flag = Peers_get_channel_flag (peer, Peers_CHANNEL_ROLE_SENDING);
+    if (GNUNET_YES == Peers_check_channel_flag (channel_flag, Peers_CHANNEL_CLEAN))
     { /* We are about to clean the sending channel. Clean the respective
        * context */
-      Peers_cleanup_destroyed_channel (cls, channel, channel_ctx);
+      Peers_cleanup_destroyed_channel (cls, channel);
       return;
     }
     else
     { /* Other peer destroyed our sending channel that he is supposed to keep
        * open. It probably went down. Remove it from our knowledge. */
-      Peers_cleanup_destroyed_channel (cls, channel, channel_ctx);
+      Peers_cleanup_destroyed_channel (cls, channel);
       remove_peer (peer);
       return;
     }
@@ -893,17 +895,18 @@ cleanup_destroyed_channel (void *cls,
      *  - ourselves  -> peer tried to establish channel twice -> clean context
      *  - other peer -> peer doesn't want to send us data -> clean
      */
+    channel_flag = Peers_get_channel_flag (peer, Peers_CHANNEL_ROLE_RECEIVING);
     if (GNUNET_YES ==
-        Peers_check_channel_flag (channel_ctx, Peers_CHANNEL_ESTABLISHED_TWICE))
+        Peers_check_channel_flag (channel_flag, Peers_CHANNEL_ESTABLISHED_TWICE))
     { /* Other peer tried to establish a channel to us twice. We do not accept
        * that. Clean the context. */
-      Peers_cleanup_destroyed_channel (cls, channel, channel_ctx);
+      Peers_cleanup_destroyed_channel (cls, channel);
       return;
     }
     else
     { /* Other peer doesn't want to send us data anymore. We are free to clean
        * it. */
-      Peers_cleanup_destroyed_channel (cls, channel, channel_ctx);
+      Peers_cleanup_destroyed_channel (cls, channel);
       clean_peer (peer);
       return;
     }
@@ -1023,7 +1026,7 @@ client_respond (void *cls,
   size_needed = sizeof (struct GNUNET_RPS_CS_ReplyMessage) +
                 num_peers * sizeof (struct GNUNET_PeerIdentity);
 
-  GNUNET_assert (GNUNET_SERVER_MAX_MESSAGE_SIZE >= size_needed);
+  GNUNET_assert (GNUNET_MAX_MESSAGE_SIZE >= size_needed);
 
   ev = GNUNET_MQ_msg_extra (out_msg,
                             num_peers * sizeof (struct GNUNET_PeerIdentity),
@@ -1048,7 +1051,6 @@ client_respond (void *cls,
  * Handle RPS request from the client.
  *
  * @param cls closure
- * @param client identification of the client
  * @param message the actual message
  */
 static void
@@ -1065,7 +1067,7 @@ handle_client_request (void *cls,
   size_needed = sizeof (struct GNUNET_RPS_CS_RequestMessage) +
                 num_peers * sizeof (struct GNUNET_PeerIdentity);
 
-  if (GNUNET_SERVER_MAX_MESSAGE_SIZE < size_needed)
+  if (GNUNET_MAX_MESSAGE_SIZE < size_needed)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Message received from client has size larger than expected\n");
@@ -1100,12 +1102,11 @@ handle_client_request (void *cls,
  * @brief Handle a message that requests the cancellation of a request
  *
  * @param cls unused
- * @param client the client that requests the cancellation
  * @param message the message containing the id of the request
  */
 static void
 handle_client_request_cancel (void *cls,
-                          const struct GNUNET_RPS_CS_RequestCancelMessage *msg)
+                              const struct GNUNET_RPS_CS_RequestCancelMessage *msg)
 {
   struct ClientContext *cli_ctx = cls;
   struct ReplyCls *rep_cls;
@@ -1157,7 +1158,6 @@ check_client_seed (void *cls, const struct GNUNET_RPS_CS_SeedMessage *msg)
  * Handle seed from the client.
  *
  * @param cls closure
- * @param client identification of the client
  * @param message the actual message
  */
 static void
@@ -1172,7 +1172,7 @@ handle_client_seed (void *cls,
   num_peers = ntohl (msg->num_peers);
   peers = (struct GNUNET_PeerIdentity *) &msg[1];
   //peers = GNUNET_new_array (num_peers, struct GNUNET_PeerIdentity);
-  //GNUNET_memcpy (peers, &in_msg[1], num_peers * sizeof (struct GNUNET_PeerIdentity));
+  //GNUNET_memcpy (peers, &msg[1], num_peers * sizeof (struct GNUNET_PeerIdentity));
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Client seeded peers:\n");
@@ -1200,18 +1200,15 @@ handle_client_seed (void *cls,
  * the channel is blocked for all other communication.
  *
  * @param cls Closure
- * @param channel The channel the CHECK was received over
- * @param channel_ctx The context associated with this channel
  * @param msg The message header
  */
-static int
+static void
 handle_peer_check (void *cls,
-                  struct GNUNET_CADET_Channel *channel,
-                  void **channel_ctx,
-                  const struct GNUNET_MessageHeader *msg)
+                   const struct GNUNET_MessageHeader *msg)
 {
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
+  const struct GNUNET_PeerIdentity *peer = cls;
+
+  GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
 }
 
 /**
@@ -1221,24 +1218,16 @@ handle_peer_check (void *cls,
  * in the temporary list for pushed PeerIDs.
  *
  * @param cls Closure
- * @param channel The channel the PUSH was received over
- * @param channel_ctx The context associated with this channel
  * @param msg The message header
  */
-static int
+static void
 handle_peer_push (void *cls,
-                  struct GNUNET_CADET_Channel *channel,
-                  void **channel_ctx,
-                  const struct GNUNET_MessageHeader *msg)
+                  const struct GNUNET_MessageHeader *msg)
 {
-  const struct GNUNET_PeerIdentity *peer;
+  const struct GNUNET_PeerIdentity *peer = cls;
 
   // (check the proof of work (?))
 
-  peer = (const struct GNUNET_PeerIdentity *)
-    GNUNET_CADET_channel_get_info (channel, GNUNET_CADET_OPTION_PEER);
-  // FIXME wait for cadet to change this function
-
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Received PUSH (%s)\n",
        GNUNET_i2s (peer));
@@ -1261,23 +1250,20 @@ handle_peer_push (void *cls,
                                    tmp_att_peer);
       add_peer_array_to_set (peer, 1, att_peer_set);
     }
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
+    GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
   }
 
 
   else if (2 == mal_type)
   { /* We attack one single well-known peer - simply ignore */
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
+    GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
   }
   #endif /* ENABLE_MALICIOUS */
 
   /* Add the sending peer to the push_map */
   CustomPeerMap_put (push_map, peer);
 
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
 }
 
 
@@ -1287,24 +1273,15 @@ handle_peer_push (void *cls,
  * Reply with the view of PeerIDs.
  *
  * @param cls Closure
- * @param channel The channel the PULL REQUEST was received over
- * @param channel_ctx The context associated with this channel
  * @param msg The message header
  */
-static int
+static void
 handle_peer_pull_request (void *cls,
-                          struct GNUNET_CADET_Channel *channel,
-                          void **channel_ctx,
-                          const struct GNUNET_MessageHeader *msg)
+                          const struct GNUNET_MessageHeader *msg)
 {
-  struct GNUNET_PeerIdentity *peer;
+  struct GNUNET_PeerIdentity *peer = cls;
   const struct GNUNET_PeerIdentity *view_array;
 
-  peer = (struct GNUNET_PeerIdentity *)
-    GNUNET_CADET_channel_get_info (channel,
-                                   GNUNET_CADET_OPTION_PEER);
-  // FIXME wait for cadet to change this function
-
   LOG (GNUNET_ERROR_TYPE_DEBUG, "Received PULL REQUEST (%s)\n", GNUNET_i2s (peer));
 
   #ifdef ENABLE_MALICIOUS
@@ -1312,8 +1289,7 @@ handle_peer_pull_request (void *cls,
       || 3 == mal_type)
   { /* Try to maximise representation */
     send_pull_reply (peer, mal_peers, num_mal_peers);
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
+    GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
   }
 
   else if (2 == mal_type)
@@ -1322,101 +1298,93 @@ handle_peer_pull_request (void *cls,
     {
       send_pull_reply (peer, mal_peers, num_mal_peers);
     }
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
+    GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
   }
   #endif /* ENABLE_MALICIOUS */
 
   view_array = View_get_as_array ();
-
   send_pull_reply (peer, view_array, View_size ());
 
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (Peers_get_recv_channel (peer));
 }
 
 
 /**
- * Handle PULL REPLY message from another peer.
- *
  * Check whether we sent a corresponding request and
  * whether this reply is the first one.
  *
  * @param cls Closure
- * @param channel The channel the PUSH was received over
- * @param channel_ctx The context associated with this channel
  * @param msg The message header
  */
 static int
-handle_peer_pull_reply (void *cls,
-                        struct GNUNET_CADET_Channel *channel,
-                        void **channel_ctx,
-                        const struct GNUNET_MessageHeader *msg)
+check_peer_pull_reply (void *cls,
+                       const struct GNUNET_RPS_P2P_PullReplyMessage *msg)
 {
-  struct GNUNET_RPS_P2P_PullReplyMessage *in_msg;
-  struct GNUNET_PeerIdentity *peers;
-  struct GNUNET_PeerIdentity *sender;
-  uint32_t i;
-#ifdef ENABLE_MALICIOUS
-  struct AttackedPeer *tmp_att_peer;
-#endif /* ENABLE_MALICIOUS */
+  struct GNUNET_PeerIdentity *sender = cls;
 
-  /* Check for protocol violation */
-  if (sizeof (struct GNUNET_RPS_P2P_PullReplyMessage) > ntohs (msg->size))
+  if (sizeof (struct GNUNET_RPS_P2P_PullReplyMessage) > ntohs (msg->header.size))
   {
     GNUNET_break_op (0);
-    GNUNET_CADET_receive_done (channel);
     return GNUNET_SYSERR;
   }
 
-  in_msg = (struct GNUNET_RPS_P2P_PullReplyMessage *) msg;
-  if ((ntohs (msg->size) - sizeof (struct GNUNET_RPS_P2P_PullReplyMessage)) /
-      sizeof (struct GNUNET_PeerIdentity) != ntohl (in_msg->num_peers))
+  if ((ntohs (msg->header.size) - sizeof (struct GNUNET_RPS_P2P_PullReplyMessage)) /
+      sizeof (struct GNUNET_PeerIdentity) != ntohl (msg->num_peers))
   {
     LOG (GNUNET_ERROR_TYPE_ERROR,
         "message says it sends %" PRIu32 " peers, have space for %lu peers\n",
-        ntohl (in_msg->num_peers),
-        (ntohs (msg->size) - sizeof (struct GNUNET_RPS_P2P_PullReplyMessage)) /
+        ntohl (msg->num_peers),
+        (ntohs (msg->header.size) - sizeof (struct GNUNET_RPS_P2P_PullReplyMessage)) /
             sizeof (struct GNUNET_PeerIdentity));
     GNUNET_break_op (0);
-    GNUNET_CADET_receive_done (channel);
     return GNUNET_SYSERR;
   }
 
-  // Guess simply casting isn't the nicest way...
-  // FIXME wait for cadet to change this function
-  sender = (struct GNUNET_PeerIdentity *)
-      GNUNET_CADET_channel_get_info (channel, GNUNET_CADET_OPTION_PEER);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Received PULL REPLY (%s)\n", GNUNET_i2s (sender));
-
   if (GNUNET_YES != Peers_check_peer_flag (sender, Peers_PULL_REPLY_PENDING))
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
         "Received a pull reply from a peer we didn't request one from!\n");
-    GNUNET_CADET_receive_done (channel);
     GNUNET_break_op (0);
-    return GNUNET_OK;
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+/**
+ * Handle PULL REPLY message from another peer.
+ *
+ * @param cls Closure
+ * @param msg The message header
+ */
+static void
+handle_peer_pull_reply (void *cls,
+                        const struct GNUNET_RPS_P2P_PullReplyMessage *msg)
+{
+  struct GNUNET_PeerIdentity *peers;
+  struct GNUNET_PeerIdentity *sender = cls;
+  uint32_t i;
+#ifdef ENABLE_MALICIOUS
+  struct AttackedPeer *tmp_att_peer;
+#endif /* ENABLE_MALICIOUS */
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Received PULL REPLY (%s)\n", GNUNET_i2s (sender));
 
   #ifdef ENABLE_MALICIOUS
   // We shouldn't even receive pull replies as we're not sending
   if (2 == mal_type)
   {
-    GNUNET_CADET_receive_done (channel);
-    return GNUNET_OK;
+    GNUNET_CADET_receive_done (Peers_get_recv_channel (sender));
   }
   #endif /* ENABLE_MALICIOUS */
 
   /* Do actual logic */
-  peers = (struct GNUNET_PeerIdentity *) &in_msg[1];
+  peers = (struct GNUNET_PeerIdentity *) &msg[1];
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "PULL REPLY received, got following %u peers:\n",
-       ntohl (in_msg->num_peers));
+       ntohl (msg->num_peers));
 
-  for (i = 0 ; i < ntohl (in_msg->num_peers) ; i++)
+  for (i = 0; i < ntohl (msg->num_peers); i++)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "%u. %s\n",
@@ -1466,8 +1434,7 @@ handle_peer_pull_reply (void *cls,
   Peers_unset_peer_flag (sender, Peers_PULL_REPLY_PENDING);
   clean_peer (sender);
 
-  GNUNET_CADET_receive_done (channel);
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (Peers_get_recv_channel (sender));
 }
 
 
@@ -2273,15 +2240,24 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *c,
      struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_CADET_MessageHandler cadet_handlers[] = {
-    {&handle_peer_check       , GNUNET_MESSAGE_TYPE_RPS_PP_CHECK_LIVE,
-      sizeof (struct GNUNET_MessageHeader)},
-    {&handle_peer_push        , GNUNET_MESSAGE_TYPE_RPS_PP_PUSH,
-      sizeof (struct GNUNET_MessageHeader)},
-    {&handle_peer_pull_request, GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REQUEST,
-      sizeof (struct GNUNET_MessageHeader)},
-    {&handle_peer_pull_reply  , GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REPLY, 0},
-    {NULL, 0, 0}
+  struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
+    GNUNET_MQ_hd_fixed_size (peer_check,
+                             GNUNET_MESSAGE_TYPE_RPS_PP_CHECK_LIVE,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (peer_push,
+                             GNUNET_MESSAGE_TYPE_RPS_PP_PUSH,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (peer_pull_request,
+                             GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REQUEST,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_var_size (peer_pull_reply,
+                           GNUNET_MESSAGE_TYPE_RPS_PP_PULL_REPLY,
+                           struct GNUNET_RPS_P2P_PullReplyMessage,
+                           NULL),
+    GNUNET_MQ_handler_end ()
   };
 
   int size;
@@ -2373,21 +2349,23 @@ run (void *cls,
 
 
   /* Initialise cadet */
-  cadet_handle = GNUNET_CADET_connect (cfg,
-                                       cls,
-                                       &cleanup_destroyed_channel,
-                                       cadet_handlers);
+  cadet_handle = GNUNET_CADET_connect (cfg);
   GNUNET_assert (NULL != cadet_handle);
   GNUNET_CRYPTO_hash (GNUNET_APPLICATION_PORT_RPS,
                       strlen (GNUNET_APPLICATION_PORT_RPS),
                       &port);
-  GNUNET_CADET_open_port (cadet_handle,
-                          &port,
-                          &Peers_handle_inbound_channel, cls);
+  cadet_port = GNUNET_CADET_open_port (cadet_handle,
+                                       &port,
+                                       &Peers_handle_inbound_channel, /* Connect handler */
+                                       NULL, /* cls */
+                                       NULL, /* WindowSize handler */
+                                       cleanup_destroyed_channel, /* Disconnect handler */
+                                       cadet_handlers);
 
 
   peerinfo_handle = GNUNET_PEERINFO_connect (cfg);
-  Peers_initialise (fn_valid_peers, cadet_handle, &own_identity);
+  Peers_initialise (fn_valid_peers, cadet_handle, cleanup_destroyed_channel,
+                    cadet_handlers, &own_identity);
   GNUNET_free (fn_valid_peers);
 
   /* Initialise sampler */
diff --git a/src/rps/gnunet-service-rps_peers.c b/src/rps/gnunet-service-rps_peers.c
index e0b278bd0..58aa84ccf 100644
--- a/src/rps/gnunet-service-rps_peers.c
+++ b/src/rps/gnunet-service-rps_peers.c
@@ -251,6 +251,17 @@ static const struct GNUNET_PeerIdentity *own_identity;
  */
 static struct GNUNET_CADET_Handle *cadet_handle;
 
+/**
+ * @brief Disconnect handler
+ */
+static GNUNET_CADET_DisconnectEventHandler cleanup_destroyed_channel;
+
+/**
+ * @brief cadet handlers
+ */
+static const struct GNUNET_MQ_MessageHandler *cadet_handlers;
+
+
 
 /**
  * @brief Get the #PeerContext associated with a peer
@@ -523,10 +534,13 @@ get_channel (const struct GNUNET_PeerIdentity *peer)
                         &port);
     peer_ctx->send_channel =
       GNUNET_CADET_channel_create (cadet_handle,
-                                   peer_ctx->send_channel_flags, /* context */
+                                   (struct GNUNET_PeerIdentity *) peer, /* context */
                                    peer,
                                    &port,
-                                   GNUNET_CADET_OPTION_RELIABLE);
+                                   GNUNET_CADET_OPTION_RELIABLE,
+                                   NULL, /* WindowSize handler */
+                                   cleanup_destroyed_channel, /* Disconnect handler */
+                                   cadet_handlers);
   }
   GNUNET_assert (NULL != peer_ctx->send_channel);
   return peer_ctx->send_channel;
@@ -552,7 +566,7 @@ get_mq (const struct GNUNET_PeerIdentity *peer)
   if (NULL == peer_ctx->mq)
   {
     (void) get_channel (peer);
-    peer_ctx->mq = GNUNET_CADET_mq_create (peer_ctx->send_channel);
+    peer_ctx->mq = GNUNET_CADET_get_mq (peer_ctx->send_channel);
   }
   return peer_ctx->mq;
 }
@@ -649,9 +663,7 @@ remove_pending_message (struct PendingMessage *pending_msg)
   GNUNET_CONTAINER_DLL_remove (peer_ctx->pending_messages_head,
                                peer_ctx->pending_messages_tail,
                                pending_msg);
-  /* FIXME We are not able to cancel messages as #GNUNET_CADET_mq_create () does
-   * not set a #GNUNET_MQ_CancelImpl */
-  /* GNUNET_MQ_send_cancel (peer_ctx->pending_messages_head->ev); */
+  GNUNET_MQ_send_cancel (peer_ctx->pending_messages_head->ev);
   GNUNET_free (pending_msg);
 }
 
@@ -932,15 +944,21 @@ restore_valid_peers ()
  *
  * @param fn_valid_peers filename of the file used to store valid peer ids
  * @param cadet_h cadet handle
+ * @param disconnect_handler Disconnect handler
+ * @param c_handlers cadet handlers
  * @param own_id own peer identity
  */
 void
 Peers_initialise (char* fn_valid_peers,
                   struct GNUNET_CADET_Handle *cadet_h,
+                  GNUNET_CADET_DisconnectEventHandler disconnect_handler,
+                  const struct GNUNET_MQ_MessageHandler *c_handlers,
                   const struct GNUNET_PeerIdentity *own_id)
 {
   filename_valid_peers = GNUNET_strdup (fn_valid_peers);
   cadet_handle = cadet_h;
+  cleanup_destroyed_channel = disconnect_handler;
+  cadet_handlers = c_handlers;
   own_identity = own_id;
   peer_map = GNUNET_CONTAINER_multipeermap_create (4, GNUNET_NO);
   valid_peers = GNUNET_CONTAINER_multipeermap_create (4, GNUNET_NO);
@@ -1279,6 +1297,34 @@ Peers_check_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags
   return check_channel_flag_set (channel_flags, flags);
 }
 
+/**
+ * @brief Get the flags for the channel in @a role for @a peer.
+ *
+ * @param peer Peer to get the channel flags for.
+ * @param role Role of channel to get flags for
+ *
+ * @return The flags.
+ */
+uint32_t *
+Peers_get_channel_flag (const struct GNUNET_PeerIdentity *peer,
+                        enum Peers_ChannelRole role)
+{
+  const struct PeerContext *peer_ctx;
+
+  peer_ctx = get_peer_ctx (peer);
+  if (Peers_CHANNEL_ROLE_SENDING == role)
+  {
+    return peer_ctx->send_channel_flags;
+  }
+  else if (Peers_CHANNEL_ROLE_RECEIVING == role)
+  {
+    return peer_ctx->recv_channel_flags;
+  }
+  else
+  {
+    GNUNET_assert (0);
+  }
+}
 
 /**
  * @brief Check whether we have information about the given peer.
@@ -1358,8 +1404,6 @@ Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer)
  * @param cls The closure
  * @param channel The channel the peer wants to establish
  * @param initiator The peer's peer ID
- * @param port The port the channel is being established over
- * @param options Further options
  *
  * @return initial channel context for the channel
  *         (can be NULL -- that's not an error)
@@ -1367,9 +1411,7 @@ Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer)
 void *
 Peers_handle_inbound_channel (void *cls,
                               struct GNUNET_CADET_Channel *channel,
-                              const struct GNUNET_PeerIdentity *initiator,
-                              const struct GNUNET_HashCode *port,
-                              enum GNUNET_CADET_ChannelOption options)
+                              const struct GNUNET_PeerIdentity *initiator)
 {
   struct PeerContext *peer_ctx;
 
@@ -1387,10 +1429,10 @@ Peers_handle_inbound_channel (void *cls,
                       Peers_CHANNEL_ESTABLISHED_TWICE);
     GNUNET_CADET_channel_destroy (channel);
     /* return the channel context */
-    return peer_ctx->recv_channel_flags;
+    return &peer_ctx->peer_id;
   }
   peer_ctx->recv_channel = channel;
-  return peer_ctx->recv_channel_flags;
+  return &peer_ctx->peer_id;
 }
 
 
@@ -1500,16 +1542,11 @@ Peers_destroy_sending_channel (const struct GNUNET_PeerIdentity *peer)
  */
 void
 Peers_cleanup_destroyed_channel (void *cls,
-                                 const struct GNUNET_CADET_Channel *channel,
-                                 void *channel_ctx)
+                                 const struct GNUNET_CADET_Channel *channel)
 {
-  struct GNUNET_PeerIdentity *peer;
+  struct GNUNET_PeerIdentity *peer = cls;
   struct PeerContext *peer_ctx;
 
-  peer = (struct GNUNET_PeerIdentity *) GNUNET_CADET_channel_get_info (
-      (struct GNUNET_CADET_Channel *) channel, GNUNET_CADET_OPTION_PEER);
-       // FIXME wait for cadet to change this function
-
   if (GNUNET_NO == Peers_check_peer_known (peer))
   {/* We don't want to implicitly create a context that we're about to kill */
   LOG (GNUNET_ERROR_TYPE_DEBUG,
@@ -1635,4 +1672,23 @@ Peers_schedule_operation (const struct GNUNET_PeerIdentity *peer,
   return GNUNET_NO;
 }
 
+/**
+ * @brief Get the recv_channel of @a peer.
+ * Needed to correctly handle (call #GNUNET_CADET_receive_done()) incoming
+ * messages.
+ *
+ * @param peer The peer to get the recv_channel from.
+ *
+ * @return The recv_channel.
+ */
+struct GNUNET_CADET_Channel *
+Peers_get_recv_channel (const struct GNUNET_PeerIdentity *peer)
+{
+  struct PeerContext *peer_ctx;
+
+  GNUNET_assert (GNUNET_YES == Peers_check_peer_known (peer));
+  peer_ctx = get_peer_ctx (peer);
+  return peer_ctx->recv_channel;
+}
+
 /* end of gnunet-service-rps_peers.c */
diff --git a/src/rps/gnunet-service-rps_peers.h b/src/rps/gnunet-service-rps_peers.h
index bbac86003..15970a7ce 100644
--- a/src/rps/gnunet-service-rps_peers.h
+++ b/src/rps/gnunet-service-rps_peers.h
@@ -117,11 +117,15 @@ typedef int
  *
  * @param fn_valid_peers filename of the file used to store valid peer ids
  * @param cadet_h cadet handle
+ * @param disconnect_handler Disconnect handler
+ * @param c_handlers cadet handlers
  * @param own_id own peer identity
  */
 void
 Peers_initialise (char* fn_valid_peers,
                   struct GNUNET_CADET_Handle *cadet_h,
+                  GNUNET_CADET_DisconnectEventHandler disconnect_handler,
+                  const struct GNUNET_MQ_MessageHandler *c_handlers,
                   const struct GNUNET_PeerIdentity *own_id);
 
 /**
@@ -259,6 +263,18 @@ int
 Peers_check_channel_flag (uint32_t *channel_flags, enum Peers_ChannelFlags flags);
 
 /**
+ * @brief Get the flags for the channel in @a role for @a peer.
+ *
+ * @param peer Peer to get the channel flags for.
+ * @param role Role of channel to get flags for
+ *
+ * @return The flags.
+ */
+uint32_t *
+Peers_get_channel_flag (const struct GNUNET_PeerIdentity *peer,
+                        enum Peers_ChannelRole role);
+
+/**
  * @brief Check whether we have information about the given peer.
  *
  * FIXME probably deprecated. Make this the new _online.
@@ -312,8 +328,6 @@ Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer);
  * @param cls The closure
  * @param channel The channel the peer wants to establish
  * @param initiator The peer's peer ID
- * @param port The port the channel is being established over
- * @param options Further options
  *
  * @return initial channel context for the channel
  *         (can be NULL -- that's not an error)
@@ -321,9 +335,7 @@ Peers_check_peer_send_intention (const struct GNUNET_PeerIdentity *peer);
 void *
 Peers_handle_inbound_channel (void *cls,
                               struct GNUNET_CADET_Channel *channel,
-                              const struct GNUNET_PeerIdentity *initiator,
-                              const struct GNUNET_HashCode *port,
-                              enum GNUNET_CADET_ChannelOption options);
+                              const struct GNUNET_PeerIdentity *initiator);
 
 /**
  * @brief Check whether a sending channel towards the given peer exists
@@ -379,8 +391,7 @@ Peers_destroy_sending_channel (const struct GNUNET_PeerIdentity *peer);
  */
 void
 Peers_cleanup_destroyed_channel (void *cls,
-                                 const struct GNUNET_CADET_Channel *channel,
-                                 void *channel_ctx);
+                                 const struct GNUNET_CADET_Channel *channel);
 
 /**
  * @brief Send a message to another peer.
@@ -411,4 +422,16 @@ int
 Peers_schedule_operation (const struct GNUNET_PeerIdentity *peer,
                           const PeerOp peer_op);
 
+/**
+ * @brief Get the recv_channel of @a peer.
+ * Needed to correctly handle (call #GNUNET_CADET_receive_done()) incoming
+ * messages.
+ *
+ * @param peer The peer to get the recv_channel from.
+ *
+ * @return The recv_channel.
+ */
+struct GNUNET_CADET_Channel *
+Peers_get_recv_channel (const struct GNUNET_PeerIdentity *peer);
+
 /* end of gnunet-service-rps_peers.h */
diff --git a/src/rps/rps_api.c b/src/rps/rps_api.c
index 504f28b92..ccd480086 100644
--- a/src/rps/rps_api.c
+++ b/src/rps/rps_api.c
@@ -389,12 +389,12 @@ GNUNET_RPS_seed_ids (struct GNUNET_RPS_Handle *h,
     n * sizeof (struct GNUNET_PeerIdentity);
   /* The number of peers that fits in one message together with
    * the respective header */
-  num_peers_max = (GNUNET_SERVER_MAX_MESSAGE_SIZE -
+  num_peers_max = (GNUNET_MAX_MESSAGE_SIZE -
       sizeof (struct GNUNET_RPS_CS_SeedMessage)) /
     sizeof (struct GNUNET_PeerIdentity);
   tmp_peer_pointer = ids;
 
-  while (GNUNET_SERVER_MAX_MESSAGE_SIZE < size_needed)
+  while (GNUNET_MAX_MESSAGE_SIZE < size_needed)
   {
     ev = GNUNET_MQ_msg_extra (msg, num_peers_max * sizeof (struct GNUNET_PeerIdentity),
         GNUNET_MESSAGE_TYPE_RPS_CS_SEED);
@@ -463,12 +463,12 @@ GNUNET_RPS_act_malicious (struct GNUNET_RPS_Handle *h,
     num_peers * sizeof (struct GNUNET_PeerIdentity);
   /* The number of peers that fit in one message together with
    * the respective header */
-  num_peers_max = (GNUNET_SERVER_MAX_MESSAGE_SIZE -
+  num_peers_max = (GNUNET_MAX_MESSAGE_SIZE -
       sizeof (struct GNUNET_RPS_CS_SeedMessage)) /
     sizeof (struct GNUNET_PeerIdentity);
   tmp_peer_pointer = peer_ids;
 
-  while (GNUNET_SERVER_MAX_MESSAGE_SIZE < size_needed)
+  while (GNUNET_MAX_MESSAGE_SIZE < size_needed)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Too many peers to send at once, sending %" PRIu32 " (all we can so far)\n",
diff --git a/src/rps/test_rps.c b/src/rps/test_rps.c
index 1ce174454..acd3a165d 100644
--- a/src/rps/test_rps.c
+++ b/src/rps/test_rps.c
@@ -546,7 +546,7 @@ seed_peers_big (void *cls)
   unsigned int i;
 
   seed_msg_size = 8; /* sizeof (struct GNUNET_RPS_CS_SeedMessage) */
-  num_peers_max = (GNUNET_SERVER_MAX_MESSAGE_SIZE - seed_msg_size) /
+  num_peers_max = (GNUNET_MAX_MESSAGE_SIZE - seed_msg_size) /
     sizeof (struct GNUNET_PeerIdentity);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
       "Peers that fit in one seed msg; %u\n",
@@ -980,7 +980,7 @@ seed_cb (struct RPSPeer *rps_peer)
 static void
 seed_big_cb (struct RPSPeer *rps_peer)
 {
-  // TODO test seeding > GNUNET_SERVER_MAX_MESSAGE_SIZE peers
+  // TODO test seeding > GNUNET_MAX_MESSAGE_SIZE peers
   GNUNET_SCHEDULER_add_delayed (
       GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 2),
       seed_peers_big, rps_peer);
@@ -1457,7 +1457,7 @@ main (int argc, char *argv[])
 
   else if (strstr (argv[0], "_seed_big") != NULL)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test seeding (num_peers > GNUNET_SERVER_MAX_MESSAGE_SIZE)\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test seeding (num_peers > GNUNET_MAX_MESSAGE_SIZE)\n");
     num_peers = 1;
     cur_test_run.name = "test-rps-seed-big";
     cur_test_run.main_test = seed_big_cb;
diff --git a/src/rps/test_service_rps_peers.c b/src/rps/test_service_rps_peers.c
index 37ed1974c..9cd677fef 100644
--- a/src/rps/test_service_rps_peers.c
+++ b/src/rps/test_service_rps_peers.c
@@ -59,25 +59,25 @@ check ()
   memset (&own_id, 1, sizeof (own_id));
 
   /* Do nothing */
-  Peers_initialise (FN_VALID_PEERS, NULL, &own_id);
+  Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id);
   Peers_terminate ();
 
 
   /* Create peer */
-  Peers_initialise (FN_VALID_PEERS, NULL, &own_id);
+  Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id);
   CHECK (GNUNET_YES == Peers_insert_peer (&k1));
   Peers_terminate ();
 
 
   /* Create peer */
-  Peers_initialise (FN_VALID_PEERS, NULL, &own_id);
+  Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id);
   CHECK (GNUNET_YES == Peers_insert_peer (&k1));
   CHECK (GNUNET_YES == Peers_remove_peer (&k1));
   Peers_terminate ();
 
 
   /* Insertion and Removal */
-  Peers_initialise (FN_VALID_PEERS, NULL, &own_id);
+  Peers_initialise (FN_VALID_PEERS, NULL, NULL, NULL, &own_id);
   CHECK (GNUNET_NO  == Peers_check_peer_known (&k1));
 
   CHECK (GNUNET_YES == Peers_insert_peer (&k1));
diff --git a/src/scalarproduct/Makefile.am b/src/scalarproduct/Makefile.am
index 98829408a..10e04284f 100644
--- a/src/scalarproduct/Makefile.am
+++ b/src/scalarproduct/Makefile.am
@@ -42,7 +42,7 @@ gnunet_service_scalarproduct_alice_SOURCES = \
   gnunet-service-scalarproduct_alice.c
 gnunet_service_scalarproduct_alice_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la \
   $(top_builddir)/src/set/libgnunetset.la \
   $(LIBGCRYPT_LIBS) \
   -lgcrypt \
@@ -53,7 +53,7 @@ gnunet_service_scalarproduct_bob_SOURCES = \
   gnunet-service-scalarproduct_bob.c
 gnunet_service_scalarproduct_bob_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la \
   $(top_builddir)/src/set/libgnunetset.la \
   $(LIBGCRYPT_LIBS) \
   -lgcrypt \
@@ -64,7 +64,7 @@ gnunet_service_scalarproduct_ecc_alice_SOURCES = \
   gnunet-service-scalarproduct-ecc_alice.c
 gnunet_service_scalarproduct_ecc_alice_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la \
   $(top_builddir)/src/set/libgnunetset.la \
   $(LIBGCRYPT_LIBS) \
   -lgcrypt \
@@ -75,7 +75,7 @@ gnunet_service_scalarproduct_ecc_bob_SOURCES = \
   gnunet-service-scalarproduct-ecc_bob.c
 gnunet_service_scalarproduct_ecc_bob_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la \
   $(top_builddir)/src/set/libgnunetset.la \
   $(LIBGCRYPT_LIBS) \
   -lgcrypt \
diff --git a/src/scalarproduct/gnunet-scalarproduct.c b/src/scalarproduct/gnunet-scalarproduct.c
index aa894b61d..5d0fce2b1 100644
--- a/src/scalarproduct/gnunet-scalarproduct.c
+++ b/src/scalarproduct/gnunet-scalarproduct.c
@@ -343,16 +343,32 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'e', "elements", "\"key1,val1;key2,val2;...,keyn,valn;\"",
-      gettext_noop ("A comma separated list of elements to compare as vector with our remote peer."),
-      1, &GNUNET_GETOPT_set_string, &input_elements},
-    {'p', "peer", "PEERID",
-      gettext_noop ("[Optional] peer to calculate our scalarproduct with. If this parameter is not given, the service will wait for a remote peer to compute the request."),
-      1, &GNUNET_GETOPT_set_string, &input_peer_id},
-    {'k', "key", "TRANSACTION_ID",
-      gettext_noop ("Transaction ID shared with peer."),
-      1, &GNUNET_GETOPT_set_string, &input_session_key},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_STRING ('e',
+                                 "elements",
+                                 "\"key1,val1;key2,val2;...,keyn,valn;\"",
+                                 gettext_noop ("A comma separated list of elements to compare as vector with our remote peer."),
+                                 &input_elements),
+
+    GNUNET_GETOPT_OPTION_STRING ('e',
+                                 "elements",
+                                 "\"key1,val1;key2,val2;...,keyn,valn;\"",
+                                 gettext_noop ("A comma separated list of elements to compare as vector with our remote peer."),
+                                 &input_elements),
+
+    GNUNET_GETOPT_OPTION_STRING ('p',
+                                 "peer",
+                                 "PEERID",
+                                 gettext_noop ("[Optional] peer to calculate our scalarproduct with. If this parameter is not given, the service will wait for a remote peer to compute the request."),
+                                 &input_peer_id),
+
+    GNUNET_GETOPT_OPTION_STRING ('k',
+                                 "key",
+                                 "TRANSACTION_ID",
+                                 gettext_noop ("Transaction ID shared with peer."),
+                                 &input_session_key),
+
     GNUNET_GETOPT_OPTION_END
   };
 
diff --git a/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c b/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c
index ca92fb9ea..c0b33f8ef 100644
--- a/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c
+++ b/src/scalarproduct/gnunet-service-scalarproduct-ecc_alice.c
@@ -842,7 +842,7 @@ client_request_complete_alice (struct AliceServiceSession *s)
               "Creating new channel for session with key %s.\n",
               GNUNET_h2s (&s->session_id));
   s->channel
-    = GNUNET_CADET_channel_creatE (my_cadet,
+    = GNUNET_CADET_channel_create (my_cadet,
                                    s,
                                    &s->peer,
                                    &s->session_id,
@@ -1173,7 +1173,7 @@ run (void *cls,
   GNUNET_CRYPTO_ecc_rnd_mpi (edc,
                              &my_privkey,
                              &my_privkey_inv);
-  my_cadet = GNUNET_CADET_connecT (cfg);
+  my_cadet = GNUNET_CADET_connect (cfg);
   if (NULL == my_cadet)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
diff --git a/src/scalarproduct/gnunet-service-scalarproduct-ecc_bob.c b/src/scalarproduct/gnunet-service-scalarproduct-ecc_bob.c
index 3851ca763..0b0333332 100644
--- a/src/scalarproduct/gnunet-service-scalarproduct-ecc_bob.c
+++ b/src/scalarproduct/gnunet-service-scalarproduct-ecc_bob.c
@@ -950,7 +950,7 @@ handle_bob_client_message (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received client request, opening port %s!\n",
               GNUNET_h2s (&msg->session_key));
-  s->port = GNUNET_CADET_open_porT (my_cadet,
+  s->port = GNUNET_CADET_open_port (my_cadet,
                                     &msg->session_key,
                                     &cb_channel_incoming,
                                     s,
@@ -1054,7 +1054,7 @@ run (void *cls,
   /* We don't really do DLOG, so we can setup with very minimal resources */
   edc = GNUNET_CRYPTO_ecc_dlog_prepare (4 /* max value */,
                                         2 /* RAM */);
-  my_cadet = GNUNET_CADET_connecT (cfg);
+  my_cadet = GNUNET_CADET_connect (cfg);
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
                                  NULL);
   if (NULL == my_cadet)
diff --git a/src/scalarproduct/gnunet-service-scalarproduct_alice.c b/src/scalarproduct/gnunet-service-scalarproduct_alice.c
index 6d7a0a3b8..a55d03900 100644
--- a/src/scalarproduct/gnunet-service-scalarproduct_alice.c
+++ b/src/scalarproduct/gnunet-service-scalarproduct_alice.c
@@ -1076,7 +1076,7 @@ client_request_complete_alice (struct AliceServiceSession *s)
               "Creating new channel for session with key %s.\n",
               GNUNET_h2s (&s->session_id));
   s->channel
-    = GNUNET_CADET_channel_creatE (my_cadet,
+    = GNUNET_CADET_channel_create (my_cadet,
                                    s,
                                    &s->peer,
                                    &s->session_id,
@@ -1398,7 +1398,7 @@ run (void *cls,
                     GNUNET_CRYPTO_PAILLIER_BITS / 3);
   GNUNET_CRYPTO_paillier_create (&my_pubkey,
                                  &my_privkey);
-  my_cadet = GNUNET_CADET_connecT (cfg);
+  my_cadet = GNUNET_CADET_connect (cfg);
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
                                  NULL);
   if (NULL == my_cadet)
diff --git a/src/scalarproduct/gnunet-service-scalarproduct_bob.c b/src/scalarproduct/gnunet-service-scalarproduct_bob.c
index f3b5327f1..0c38cb426 100644
--- a/src/scalarproduct/gnunet-service-scalarproduct_bob.c
+++ b/src/scalarproduct/gnunet-service-scalarproduct_bob.c
@@ -1230,7 +1230,7 @@ handle_bob_client_message (void *cls,
   }
   GNUNET_SERVICE_client_continue (s->client);
   /* We're ready, open the port */
-  s->port = GNUNET_CADET_open_porT (my_cadet,
+  s->port = GNUNET_CADET_open_port (my_cadet,
                                     &msg->session_key,
                                     &cb_channel_incoming,
                                     s,
@@ -1336,7 +1336,7 @@ run (void *cls,
 
   GNUNET_CRYPTO_paillier_create (&my_pubkey,
                                  &my_privkey);
-  my_cadet = GNUNET_CADET_connecT (cfg);
+  my_cadet = GNUNET_CADET_connect (cfg);
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
                                  NULL);
   if (NULL == my_cadet)
diff --git a/src/scalarproduct/scalarproduct_api.c b/src/scalarproduct/scalarproduct_api.c
index df9f8d196..05c122e74 100644
--- a/src/scalarproduct/scalarproduct_api.c
+++ b/src/scalarproduct/scalarproduct_api.c
@@ -268,7 +268,7 @@ GNUNET_SCALARPRODUCT_accept_computation (const struct GNUNET_CONFIGURATION_Handl
     GNUNET_free (h);
     return NULL;
   }
-  possible = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - sizeof (struct BobComputationMessage))
+  possible = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof (struct BobComputationMessage))
     / sizeof (struct GNUNET_SCALARPRODUCT_Element);
   todo = GNUNET_MIN (possible,
                      element_count);
@@ -285,7 +285,7 @@ GNUNET_SCALARPRODUCT_accept_computation (const struct GNUNET_CONFIGURATION_Handl
   element_count_transfered = todo;
   GNUNET_MQ_send (h->mq,
                   env);
-  possible = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - sizeof (*mmsg))
+  possible = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof (*mmsg))
     / sizeof (struct GNUNET_SCALARPRODUCT_Element);
   while (element_count_transfered < element_count)
   {
@@ -426,7 +426,7 @@ GNUNET_SCALARPRODUCT_start_computation (const struct GNUNET_CONFIGURATION_Handle
   h->cfg = cfg;
   h->key = *session_key;
 
-  possible = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - sizeof (struct AliceComputationMessage))
+  possible = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof (struct AliceComputationMessage))
       / sizeof (struct GNUNET_SCALARPRODUCT_Element);
   todo = GNUNET_MIN (possible,
                      element_count);
@@ -445,7 +445,7 @@ GNUNET_SCALARPRODUCT_start_computation (const struct GNUNET_CONFIGURATION_Handle
   GNUNET_MQ_send (h->mq,
                   env);
   element_count_transfered = todo;
-  possible = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - sizeof (*mmsg))
+  possible = (GNUNET_MAX_MESSAGE_SIZE - 1 - sizeof (*mmsg))
     / sizeof (struct GNUNET_SCALARPRODUCT_Element);
   while (element_count_transfered < element_count)
   {
diff --git a/src/secretsharing/Makefile.am b/src/secretsharing/Makefile.am
index 485183e36..4b9a06c40 100644
--- a/src/secretsharing/Makefile.am
+++ b/src/secretsharing/Makefile.am
@@ -17,9 +17,6 @@ if USE_COVERAGE
 endif
 
 
-bin_PROGRAMS = \
- gnunet-secretsharing-profiler
-
 libexec_PROGRAMS = \
  gnunet-service-secretsharing
 
@@ -57,6 +54,9 @@ libgnunetsecretsharing_la_LDFLAGS = \
   $(GN_LIB_LDFLAGS)
 
 if HAVE_TESTING
+bin_PROGRAMS = \
+ gnunet-secretsharing-profiler
+
 check_PROGRAMS = \
  test_secretsharing_api
 
diff --git a/src/secretsharing/gnunet-secretsharing-profiler.c b/src/secretsharing/gnunet-secretsharing-profiler.c
index 3ff5d7fdd..e66019dc6 100644
--- a/src/secretsharing/gnunet-secretsharing-profiler.c
+++ b/src/secretsharing/gnunet-secretsharing-profiler.c
@@ -41,7 +41,7 @@ static unsigned int threshold = 2;
 /**
  * Should we try to decrypt a value after the key generation?
  */
-static unsigned int decrypt = GNUNET_NO;
+static int decrypt = GNUNET_NO;
 
 /**
  * When would we like to see the operation finished?
@@ -88,7 +88,7 @@ static unsigned int num_decrypted;
 
 static struct GNUNET_HashCode session_id;
 
-static int verbose;
+static unsigned int verbose;
 
 static struct GNUNET_SECRETSHARING_Plaintext reference_plaintext;
 
@@ -602,26 +602,41 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char **argv)
 {
-   static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-      { 'n', "num-peers", NULL,
-        gettext_noop ("number of peers in consensus"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_peers },
-      { 'D', "delay", NULL,
-        gettext_noop ("dkg start delay"),
-        GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &delay },
-      { 't', "timeout", NULL,
-        gettext_noop ("dkg timeout"),
-        GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &timeout },
-      { 'k', "threshold", NULL,
-        gettext_noop ("threshold"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &threshold },
-      { 'd', "decrypt", NULL,
-        gettext_noop ("also profile decryption"),
-        GNUNET_NO, &GNUNET_GETOPT_set_one, &decrypt },
-      { 'V', "verbose", NULL,
-        gettext_noop ("be more verbose (print received values)"),
-        GNUNET_NO, &GNUNET_GETOPT_set_one, &verbose },
-      GNUNET_GETOPT_OPTION_END
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('n',
+                                   "num-peers",
+                                   NULL,
+                                   gettext_noop ("number of peers in consensus"),
+                                   &num_peers),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('D',
+                                            "delay",
+                                            NULL,
+                                            gettext_noop ("dkg start delay"),
+                                            &delay),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('t',
+                                            "timeout",
+                                            NULL,
+                                            gettext_noop ("dkg timeout"),
+                                            &timeout),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('k',
+                                   "threshold",
+                                   NULL,
+                                   gettext_noop ("threshold"),
+                                   &threshold),
+    
+    GNUNET_GETOPT_OPTION_SET_ONE ('d',
+                                  "descrypt",
+                                  gettext_noop ("also profile decryption"),
+                                  &decrypt),
+
+
+    GNUNET_GETOPT_OPTION_VERBOSE (&verbose),
+
+    GNUNET_GETOPT_OPTION_END
   };
   delay = GNUNET_TIME_UNIT_ZERO;
   timeout = GNUNET_TIME_UNIT_MINUTES;
diff --git a/src/secretsharing/gnunet-service-secretsharing.c b/src/secretsharing/gnunet-service-secretsharing.c
index ddbe81576..ccdba12c2 100644
--- a/src/secretsharing/gnunet-service-secretsharing.c
+++ b/src/secretsharing/gnunet-service-secretsharing.c
@@ -109,19 +109,16 @@ struct DecryptPeerInfo
 
 
 /**
+ * State we keep per client.
+ */
+struct ClientState;
+
+
+/**
  * Session to establish a threshold-shared secret.
  */
 struct KeygenSession
 {
-  /**
-   * Keygen sessions are held in a linked list.
-   */
-  struct KeygenSession *next;
-
-  /**
-   * Keygen sessions are held in a linked list.
-   */
-  struct KeygenSession *prev;
 
   /**
    * Current consensus, used for both DKG rounds.
@@ -129,15 +126,9 @@ struct KeygenSession
   struct GNUNET_CONSENSUS_Handle *consensus;
 
   /**
-   * Client that is interested in the result
-   * of this key generation session.
-   */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * Message queue for 'client'
+   * Which client is this for?
    */
-  struct GNUNET_MQ_Handle *client_mq;
+  struct ClientState *cs;
 
   /**
    * Randomly generated coefficients of the polynomial for sharing our
@@ -223,15 +214,6 @@ struct KeygenSession
  */
 struct DecryptSession
 {
-  /**
-   * Decrypt sessions are stored in a linked list.
-   */
-  struct DecryptSession *next;
-
-  /**
-   * Decrypt sessions are stored in a linked list.
-   */
-  struct DecryptSession *prev;
 
   /**
    * Handle to the consensus over partial decryptions.
@@ -239,14 +221,9 @@ struct DecryptSession
   struct GNUNET_CONSENSUS_Handle *consensus;
 
   /**
-   * Client connected to us.
+   * Which client is this for?
    */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * Message queue for 'client'.
-   */
-  struct GNUNET_MQ_Handle *client_mq;
+  struct ClientState *cs;
 
   /**
    * When should we start communicating for decryption?
@@ -279,24 +256,32 @@ struct DecryptSession
 
 
 /**
- * Decrypt sessions are held in a linked list.
+ * State we keep per client.
  */
-static struct DecryptSession *decrypt_sessions_head;
+struct ClientState
+{
+  /**
+   * Decrypt session of the client, if any.
+   */
+  struct DecryptSession *decrypt_session;
 
-/**
- * Decrypt sessions are held in a linked list.
- */
-static struct DecryptSession *decrypt_sessions_tail;
+  /**
+   * Keygen session of the client, if any.
+   */
+  struct KeygenSession *keygen_session;
 
-/**
- * Decrypt sessions are held in a linked list.
- */
-static struct KeygenSession *keygen_sessions_head;
+  /**
+   * Client this is about.
+   */
+  struct GNUNET_SERVICE_Client *client;
+
+  /**
+   * MQ to talk to @a client.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+};
 
-/**
- * Decrypt sessions are held in a linked list.
- */
-static struct KeygenSession *keygen_sessions_tail;
 
 /**
  * The ElGamal prime field order as libgcrypt mpi.
@@ -331,11 +316,6 @@ static struct GNUNET_CRYPTO_EddsaPrivateKey *my_peer_private_key;
  */
 static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
-/**
- * Server for this service.
- */
-static struct GNUNET_SERVER_Handle *srv;
-
 
 /**
  * Get the peer info belonging to a peer identity in a keygen session.
@@ -468,7 +448,8 @@ normalize_peers (struct GNUNET_PeerIdentity *listed,
     n += 1;
   }
 
-  normalized = GNUNET_new_array (n, struct GNUNET_PeerIdentity);
+  normalized = GNUNET_new_array (n,
+                                 struct GNUNET_PeerIdentity);
 
   if (GNUNET_NO == local_peer_in_list)
     normalized[n - 1] = my_peer;
@@ -558,10 +539,13 @@ compute_lagrange_coefficient (gcry_mpi_t coeff, unsigned int j,
 static void
 decrypt_session_destroy (struct DecryptSession *ds)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "destroying decrypt session\n");
-
-  GNUNET_CONTAINER_DLL_remove (decrypt_sessions_head, decrypt_sessions_tail, ds);
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "destroying decrypt session\n");
+  if (NULL != ds->cs)
+  {
+    ds->cs->decrypt_session = NULL;
+    ds->cs = NULL;
+  }
   if (NULL != ds->consensus)
   {
     GNUNET_CONSENSUS_destroy (ds->consensus);
@@ -570,8 +554,7 @@ decrypt_session_destroy (struct DecryptSession *ds)
 
   if (NULL != ds->info)
   {
-    unsigned int i;
-    for (i = 0; i < ds->share->num_peers; i++)
+    for (unsigned int i = 0; i < ds->share->num_peers; i++)
     {
       if (NULL != ds->info[i].partial_decryption)
       {
@@ -582,26 +565,12 @@ decrypt_session_destroy (struct DecryptSession *ds)
     GNUNET_free (ds->info);
     ds->info = NULL;
   }
-
   if (NULL != ds->share)
   {
     GNUNET_SECRETSHARING_share_destroy (ds->share);
     ds->share = NULL;
   }
 
-  if (NULL != ds->client_mq)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "destroying decrypt MQ\n");
-    GNUNET_MQ_destroy (ds->client_mq);
-    ds->client_mq = NULL;
-  }
-
-  if (NULL != ds->client)
-  {
-    GNUNET_SERVER_client_disconnect (ds->client);
-    ds->client = NULL;
-  }
-
   GNUNET_free (ds);
 }
 
@@ -630,14 +599,17 @@ keygen_info_destroy (struct KeygenPeerInfo *info)
 static void
 keygen_session_destroy (struct KeygenSession *ks)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "destroying keygen session\n");
-
-  GNUNET_CONTAINER_DLL_remove (keygen_sessions_head, keygen_sessions_tail, ks);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "destroying keygen session\n");
 
+  if (NULL != ks->cs)
+  {
+    ks->cs->keygen_session = NULL;
+    ks->cs = NULL;
+  }
   if (NULL != ks->info)
   {
-    unsigned int i;
-    for (i = 0; i < ks->num_peers; i++)
+    for (unsigned int i = 0; i < ks->num_peers; i++)
       keygen_info_destroy (&ks->info[i]);
     GNUNET_free (ks->info);
     ks->info = NULL;
@@ -651,8 +623,7 @@ keygen_session_destroy (struct KeygenSession *ks)
 
   if (NULL != ks->presecret_polynomial)
   {
-    unsigned int i;
-    for (i = 0; i < ks->threshold; i++)
+    for (unsigned int i = 0; i < ks->threshold; i++)
     {
       GNUNET_assert (NULL != ks->presecret_polynomial[i]);
       gcry_mpi_release (ks->presecret_polynomial[i]);
@@ -661,38 +632,21 @@ keygen_session_destroy (struct KeygenSession *ks)
     GNUNET_free (ks->presecret_polynomial);
     ks->presecret_polynomial = NULL;
   }
-
-  if (NULL != ks->client_mq)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "destroying keygen MQ\n");
-    GNUNET_MQ_destroy (ks->client_mq);
-    ks->client_mq = NULL;
-  }
-
   if (NULL != ks->my_share)
   {
     gcry_mpi_release (ks->my_share);
     ks->my_share = NULL;
   }
-
   if (NULL != ks->public_key)
   {
     gcry_mpi_release (ks->public_key);
     ks->public_key = NULL;
   }
-
   if (NULL != ks->peers)
   {
     GNUNET_free (ks->peers);
     ks->peers = NULL;
   }
-
-  if (NULL != ks->client)
-  {
-    GNUNET_SERVER_client_disconnect (ks->client);
-    ks->client = NULL;
-  }
-
   GNUNET_free (ks);
 }
 
@@ -706,11 +660,7 @@ keygen_session_destroy (struct KeygenSession *ks)
 static void
 cleanup_task (void *cls)
 {
-  while (NULL != decrypt_sessions_head)
-    decrypt_session_destroy (decrypt_sessions_head);
-
-  while (NULL != keygen_sessions_head)
-    keygen_session_destroy (keygen_sessions_head);
+  /* Nothing to do! */
 }
 
 
@@ -727,7 +677,8 @@ generate_presecret_polynomial (struct KeygenSession *ks)
   gcry_mpi_t v;
 
   GNUNET_assert (NULL == ks->presecret_polynomial);
-  ks->presecret_polynomial = GNUNET_new_array (ks->threshold, gcry_mpi_t);
+  ks->presecret_polynomial = GNUNET_new_array (ks->threshold,
+                                               gcry_mpi_t);
   for (i = 0; i < ks->threshold; i++)
   {
     v = ks->presecret_polynomial[i] = gcry_mpi_new (GNUNET_SECRETSHARING_ELGAMAL_BITS);
@@ -768,9 +719,9 @@ keygen_round1_new_element (void *cls,
   if (element->size != sizeof (struct GNUNET_SECRETSHARING_KeygenCommitData))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "keygen commit data with wrong size (%u) in consensus, "
-                " %lu expected\n",
-                element->size, sizeof (struct GNUNET_SECRETSHARING_KeygenCommitData));
+                "keygen commit data with wrong size (%u) in consensus, %u expected\n",
+                (unsigned int) element->size,
+                (unsigned int) sizeof (struct GNUNET_SECRETSHARING_KeygenCommitData));
     return;
   }
 
@@ -855,10 +806,13 @@ keygen_round2_conclude (void *cls)
     if (GNUNET_YES == ks->info[i].round2_valid)
       share->num_peers++;
 
-  share->peers = GNUNET_new_array (share->num_peers, struct GNUNET_PeerIdentity);
+  share->peers = GNUNET_new_array (share->num_peers,
+                                   struct GNUNET_PeerIdentity);
   share->sigmas =
-      GNUNET_new_array (share->num_peers, struct GNUNET_SECRETSHARING_FieldElement);
-  share->original_indices = GNUNET_new_array (share->num_peers, uint16_t);
+      GNUNET_new_array (share->num_peers,
+                        struct GNUNET_SECRETSHARING_FieldElement);
+  share->original_indices = GNUNET_new_array (share->num_peers,
+                                              uint16_t);
 
   /* maybe we're not even in the list of peers? */
   share->my_peer = share->num_peers;
@@ -907,7 +861,8 @@ keygen_round2_conclude (void *cls)
   GNUNET_SECRETSHARING_share_destroy (share);
   share = NULL;
 
-  GNUNET_MQ_send (ks->client_mq, ev);
+  GNUNET_MQ_send (ks->cs->mq,
+                  ev);
 }
 
 
@@ -1429,9 +1384,9 @@ keygen_round2_new_element (void *cls,
   if (element->size != expected_element_size)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "keygen round2 data with wrong size (%u) in consensus, "
-                " %lu expected\n",
-                element->size, expected_element_size);
+                "keygen round2 data with wrong size (%u) in consensus, %u expected\n",
+                (unsigned int) element->size,
+                (unsigned int) expected_element_size);
     return;
   }
 
@@ -1669,73 +1624,102 @@ insert_round1_element (struct KeygenSession *ks)
 
 
 /**
+ * Check that @a msg is well-formed.
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ * @return #GNUNET_OK if @a msg is well-formed
+ */
+static int
+check_client_keygen (void *cls,
+                     const struct GNUNET_SECRETSHARING_CreateMessage *msg)
+{
+  unsigned int num_peers = ntohs (msg->num_peers);
+
+  if (ntohs (msg->header.size) - sizeof (*msg) !=
+      num_peers * sizeof (struct GNUNET_PeerIdentity))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
  * Functions with this signature are called whenever a message is
  * received.
  *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
+ * @param cls identification of the client
+ * @param msg the actual message
  */
-static void handle_client_keygen (void *cls,
-                                  struct GNUNET_SERVER_Client *client,
-                                  const struct GNUNET_MessageHeader
-                                  *message)
+static void
+handle_client_keygen (void *cls,
+                      const struct GNUNET_SECRETSHARING_CreateMessage *msg)
 {
-  const struct GNUNET_SECRETSHARING_CreateMessage *msg =
-      (const struct GNUNET_SECRETSHARING_CreateMessage *) message;
+  struct ClientState *cs = cls;
   struct KeygenSession *ks;
-  unsigned int i;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "client requested key generation\n");
 
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "client requested key generation\n");
+  if (NULL != cs->keygen_session)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (cs->client);
+    return;
+  }
   ks = GNUNET_new (struct KeygenSession);
-
-  /* FIXME: check if client already has some session */
-
-  GNUNET_CONTAINER_DLL_insert (keygen_sessions_head, keygen_sessions_tail, ks);
-
-  ks->client = client;
-  ks->client_mq = GNUNET_MQ_queue_for_server_client (client);
-
+  ks->cs = cs;
+  cs->keygen_session = ks;
   ks->deadline = GNUNET_TIME_absolute_ntoh (msg->deadline);
   ks->threshold = ntohs (msg->threshold);
   ks->num_peers = ntohs (msg->num_peers);
 
-  ks->peers = normalize_peers ((struct GNUNET_PeerIdentity *) &msg[1], ks->num_peers,
-                               &ks->num_peers, &ks->local_peer_idx);
+  ks->peers = normalize_peers ((struct GNUNET_PeerIdentity *) &msg[1],
+                               ks->num_peers,
+                               &ks->num_peers,
+                               &ks->local_peer_idx);
 
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "first round of consensus with %u peers\n", ks->num_peers);
-  ks->consensus = GNUNET_CONSENSUS_create (cfg, ks->num_peers, ks->peers, &msg->session_id,
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "first round of consensus with %u peers\n",
+              ks->num_peers);
+  ks->consensus = GNUNET_CONSENSUS_create (cfg,
+                                           ks->num_peers,
+                                           ks->peers,
+                                           &msg->session_id,
                                            GNUNET_TIME_absolute_ntoh (msg->start),
                                            GNUNET_TIME_absolute_ntoh (msg->deadline),
-                                           keygen_round1_new_element, ks);
+                                           keygen_round1_new_element,
+                                           ks);
 
-  ks->info = GNUNET_new_array (ks->num_peers, struct KeygenPeerInfo);
+  ks->info = GNUNET_new_array (ks->num_peers,
+                               struct KeygenPeerInfo);
 
-  for (i = 0; i < ks->num_peers; i++)
+  for (unsigned int i = 0; i < ks->num_peers; i++)
     ks->info[i].peer = ks->peers[i];
 
   GNUNET_CRYPTO_paillier_create (&ks->info[ks->local_peer_idx].paillier_public_key,
                                  &ks->paillier_private_key);
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: Generated paillier key pair\n", ks->local_peer_idx);
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "P%u: Generated paillier key pair\n",
+              ks->local_peer_idx);
   generate_presecret_polynomial (ks);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: Generated presecret polynomial\n", ks->local_peer_idx);
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "P%u: Generated presecret polynomial\n",
+              ks->local_peer_idx);
   insert_round1_element (ks);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: Concluding for round 1\n", ks->local_peer_idx);
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "P%u: Concluding for round 1\n",
+              ks->local_peer_idx);
   GNUNET_CONSENSUS_conclude (ks->consensus,
                              keygen_round1_conclude,
                              ks);
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "P%u: Waiting for round 1 elements ...\n", ks->local_peer_idx);
+  GNUNET_SERVICE_client_continue (cs->client);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "P%u: Waiting for round 1 elements ...\n",
+              ks->local_peer_idx);
 }
 
 
@@ -1771,20 +1755,24 @@ decrypt_conclude (void *cls)
     if (NULL != ds->info[i].partial_decryption)
       num++;
 
-  indices = GNUNET_malloc (num * sizeof (unsigned int));
+  indices = GNUNET_new_array (num,
+                              unsigned int);
   j = 0;
   for (i = 0; i < ds->share->num_peers; i++)
     if (NULL != ds->info[i].partial_decryption)
       indices[j++] = ds->info[i].original_index;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "P%u: decrypt conclude, with %u peers\n",
-              ds->share->my_peer, num);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "P%u: decrypt conclude, with %u peers\n",
+              ds->share->my_peer,
+              num);
 
   gcry_mpi_set_ui (prod, 1);
   for (i = 0; i < num; i++)
   {
 
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "P%u: index of %u: %u\n",
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "P%u: index of %u: %u\n",
                 ds->share->my_peer, i, indices[i]);
     compute_lagrange_coefficient (lagrange, indices[i], indices, num);
     // w_i^{\lambda_i}
@@ -1801,7 +1789,8 @@ decrypt_conclude (void *cls)
   ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SECRETSHARING_CLIENT_DECRYPT_DONE);
   GNUNET_CRYPTO_mpi_print_unsigned (&msg->plaintext, GNUNET_SECRETSHARING_ELGAMAL_BITS / 8, m);
   msg->success = htonl (1);
-  GNUNET_MQ_send (ds->client_mq, ev);
+  GNUNET_MQ_send (ds->cs->mq,
+                  ev);
 
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, "sent decrypt done to client\n");
 
@@ -1945,10 +1934,15 @@ decrypt_new_element (void *cls,
   {
     char *tmp1_str;
     char *tmp2_str;
+
     tmp1_str = mpi_to_str (tmp1);
     tmp2_str = mpi_to_str (tmp2);
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "P%u: Received invalid partial decryption from P%ld (eqn 1), expected %s got %s\n",
-                session->share->my_peer, info - session->info, tmp1_str, tmp2_str);
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "P%u: Received invalid partial decryption from P%u (eqn 1), expected %s got %s\n",
+                session->share->my_peer,
+                (unsigned int) (info - session->info),
+                tmp1_str,
+                tmp2_str);
     GNUNET_free (tmp1_str);
     GNUNET_free (tmp2_str);
     goto cleanup;
@@ -1963,8 +1957,10 @@ decrypt_new_element (void *cls,
 
   if (0 != gcry_mpi_cmp (tmp1, tmp2))
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "P%u: Received invalid partial decryption from P%ld (eqn 2)\n",
-                session->share->my_peer, info - session->info);
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "P%u: Received invalid partial decryption from P%u (eqn 2)\n",
+                session->share->my_peer,
+                (unsigned int) (info - session->info));
     goto cleanup;
   }
 
@@ -2100,40 +2096,63 @@ insert_decrypt_element (struct DecryptSession *ds)
 
 
 /**
+ * Check that @a msg is well-formed.
+ *
+ * @param cls identification of the client
+ * @param msg the actual message
+ * @return #GNUNET_OK (check deferred a bit)
+ */
+static int
+check_client_decrypt (void *cls,
+                      const struct GNUNET_SECRETSHARING_DecryptRequestMessage *msg)
+{
+  /* we check later, it's complicated */
+  return GNUNET_OK;
+}
+
+
+/**
  * Functions with this signature are called whenever a message is
  * received.
  *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
+ * @param cls identification of the client
+ * @param msg the actual message
  */
-static void handle_client_decrypt (void *cls,
-                                   struct GNUNET_SERVER_Client *client,
-                                   const struct GNUNET_MessageHeader
-                                   *message)
+static void
+handle_client_decrypt (void *cls,
+                       const struct GNUNET_SECRETSHARING_DecryptRequestMessage *msg)
 {
-  const struct GNUNET_SECRETSHARING_DecryptRequestMessage *msg =
-      (const void *) message;
+  struct ClientState *cs = cls;
   struct DecryptSession *ds;
   struct GNUNET_HashCode session_id;
-  unsigned int i;
 
+  if (NULL != cs->decrypt_session)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (cs->client);
+    return;
+  }
   ds = GNUNET_new (struct DecryptSession);
-  // FIXME: check if session already exists
-  GNUNET_CONTAINER_DLL_insert (decrypt_sessions_head, decrypt_sessions_tail, ds);
-  ds->client = client;
-  ds->client_mq = GNUNET_MQ_queue_for_server_client (client);
+  cs->decrypt_session = ds;
+  ds->cs = cs;
   ds->start = GNUNET_TIME_absolute_ntoh (msg->start);
   ds->deadline = GNUNET_TIME_absolute_ntoh (msg->deadline);
   ds->ciphertext = msg->ciphertext;
 
-  ds->share = GNUNET_SECRETSHARING_share_read (&msg[1], ntohs (msg->header.size) - sizeof *msg, NULL);
-  // FIXME: probably should be break rather than assert
-  GNUNET_assert (NULL != ds->share);
-
-  // FIXME: this is probably sufficient, but kdf/hash with all values would be nicer ...
-  GNUNET_CRYPTO_hash (&msg->ciphertext, sizeof (struct GNUNET_SECRETSHARING_Ciphertext), &session_id);
+  ds->share = GNUNET_SECRETSHARING_share_read (&msg[1],
+                                               ntohs (msg->header.size) - sizeof (*msg),
+                                               NULL);
+  if (NULL == ds->share)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (cs->client);
+    return;
+  }
 
+  /* FIXME: this is probably sufficient, but kdf/hash with all values would be nicer ... */
+  GNUNET_CRYPTO_hash (&msg->ciphertext,
+                      sizeof (struct GNUNET_SECRETSHARING_Ciphertext),
+                      &session_id);
   ds->consensus = GNUNET_CONSENSUS_create (cfg,
                                            ds->share->num_peers,
                                            ds->share->peers,
@@ -2144,20 +2163,20 @@ static void handle_client_decrypt (void *cls,
                                            ds);
 
 
-  ds->info = GNUNET_new_array (ds->share->num_peers, struct DecryptPeerInfo);
-  for (i = 0; i < ds->share->num_peers; i++)
+  ds->info = GNUNET_new_array (ds->share->num_peers,
+                               struct DecryptPeerInfo);
+  for (unsigned int i = 0; i < ds->share->num_peers; i++)
   {
     ds->info[i].peer = ds->share->peers[i];
     ds->info[i].original_index = ds->share->original_indices[i];
   }
-
   insert_decrypt_element (ds);
-
-  GNUNET_CONSENSUS_conclude (ds->consensus, decrypt_conclude, ds);
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "decrypting with %u peers\n",
+  GNUNET_CONSENSUS_conclude (ds->consensus,
+                             decrypt_conclude,
+                             ds);
+  GNUNET_SERVICE_client_continue (cs->client);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "decrypting with %u peers\n",
               ds->share->num_peers);
 }
 
@@ -2174,103 +2193,104 @@ init_crypto_constants (void)
 }
 
 
-static struct KeygenSession *
-keygen_session_get (struct GNUNET_SERVER_Client *client)
-{
-  struct KeygenSession *ks;
-  for (ks = keygen_sessions_head; NULL != ks; ks = ks->next)
-    if (ks->client == client)
-      return ks;
-  return NULL;
-}
-
-static struct DecryptSession *
-decrypt_session_get (struct GNUNET_SERVER_Client *client)
-{
-  struct DecryptSession *ds;
-  for (ds = decrypt_sessions_head; NULL != ds; ds = ds->next)
-    if (ds->client == client)
-      return ds;
-  return NULL;
-}
-
-
-/**
- * Clean up after a client has disconnected
- *
- * @param cls closure, unused
- * @param client the client to clean up after
- */
-static void
-handle_client_disconnect (void *cls, struct GNUNET_SERVER_Client *client)
-{
-  struct KeygenSession *ks;
-  struct DecryptSession *ds;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "handling client disconnect\n");
-
-  ks = keygen_session_get (client);
-  if (NULL != ks)
-    keygen_session_destroy (ks);
-
-  ds = decrypt_session_get (client);
-  if (NULL != ds)
-    decrypt_session_destroy (ds);
-}
-
-
 /**
- * Process template requests.
+ * Initialize secretsharing service.
  *
  * @param cls closure
- * @param server the initialized server
  * @param c configuration to use
+ * @param service the initialized service
  */
 static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
-     const struct GNUNET_CONFIGURATION_Handle *c)
+run (void *cls,
+     const struct GNUNET_CONFIGURATION_Handle *c,
+     struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_SERVER_MessageHandler handlers[] = {
-    {handle_client_keygen, NULL, GNUNET_MESSAGE_TYPE_SECRETSHARING_CLIENT_GENERATE, 0},
-    {handle_client_decrypt, NULL, GNUNET_MESSAGE_TYPE_SECRETSHARING_CLIENT_DECRYPT, 0},
-    {NULL, NULL, 0, 0}
-  };
   cfg = c;
-  srv = server;
   my_peer_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (c);
   if (NULL == my_peer_private_key)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not access host private key\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "could not access host private key\n");
     GNUNET_break (0);
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
   init_crypto_constants ();
-  if (GNUNET_OK != GNUNET_CRYPTO_get_peer_identity (cfg, &my_peer))
+  if (GNUNET_OK !=
+      GNUNET_CRYPTO_get_peer_identity (cfg,
+                                       &my_peer))
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not retrieve host identity\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "could not retrieve host identity\n");
     GNUNET_break (0);
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  GNUNET_SERVER_add_handlers (server, handlers);
-  GNUNET_SERVER_disconnect_notify (server, &handle_client_disconnect, NULL);
   GNUNET_SCHEDULER_add_shutdown (&cleanup_task,
                                  NULL);
 }
 
 
 /**
- * The main function for the template service.
+ * Callback called when a client connects to the service.
+ *
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
+ */
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
+{
+  struct ClientState *cs = GNUNET_new (struct ClientState);;
+
+  cs->client = c;
+  cs->mq = mq;
+  return cs;
+}
+
+
+/**
+ * Callback called when a client disconnected from the service
  *
- * @param argc number of arguments from the command line
- * @param argv command line arguments
- * @return 0 ok, 1 on error
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls should be equal to @a c
  */
-int
-main (int argc, char *const *argv)
+static void
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *c,
+                      void *internal_cls)
 {
-  return (GNUNET_OK ==
-          GNUNET_SERVICE_run (argc, argv, "secretsharing",
-                              GNUNET_SERVICE_OPTION_NONE, &run, NULL)) ? 0 : 1;
+  struct ClientState *cs = internal_cls;
+
+  if (NULL != cs->keygen_session)
+    keygen_session_destroy (cs->keygen_session);
+
+  if (NULL != cs->decrypt_session)
+    decrypt_session_destroy (cs->decrypt_session);
+  GNUNET_free (cs);
 }
+
+
+/**
+ * Define "main" method using service macro.
+ */
+GNUNET_SERVICE_MAIN
+("secretsharing",
+ GNUNET_SERVICE_OPTION_NONE,
+ &run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_var_size (client_keygen,
+                        GNUNET_MESSAGE_TYPE_SECRETSHARING_CLIENT_GENERATE,
+                        struct GNUNET_SECRETSHARING_CreateMessage,
+                        NULL),
+ GNUNET_MQ_hd_var_size (client_decrypt,
+                        GNUNET_MESSAGE_TYPE_SECRETSHARING_CLIENT_DECRYPT,
+                        struct GNUNET_SECRETSHARING_DecryptRequestMessage,
+                        NULL),
+ GNUNET_MQ_handler_end ());
diff --git a/src/set/Makefile.am b/src/set/Makefile.am
index cfe95bc1a..14667d0ef 100644
--- a/src/set/Makefile.am
+++ b/src/set/Makefile.am
@@ -51,7 +51,7 @@ gnunet_set_ibf_profiler_LDADD = \
 
 gnunet_service_set_SOURCES = \
  gnunet-service-set.c gnunet-service-set.h \
- gnunet-service-set_union.c \
+ gnunet-service-set_union.c gnunet-service-set_union.h \
  gnunet-service-set_intersection.c \
  ibf.c ibf.h \
  gnunet-service-set_union_strata_estimator.c gnunet-service-set_union_strata_estimator.h \
@@ -60,7 +60,7 @@ gnunet_service_set_LDADD = \
   $(top_builddir)/src/util/libgnunetutil.la \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(top_builddir)/src/core/libgnunetcore.la \
-  $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la \
   $(top_builddir)/src/block/libgnunetblock.la \
   libgnunetset.la \
   $(GN_LIBINTL)
diff --git a/src/set/gnunet-service-set.c b/src/set/gnunet-service-set.c
index 30d43e8a1..12af653c1 100644
--- a/src/set/gnunet-service-set.c
+++ b/src/set/gnunet-service-set.c
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2013, 2014, 2017 GNUnet e.V.
+      Copyright (C) 2013-2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -24,6 +24,8 @@
  * @author Christian Grothoff
  */
 #include "gnunet-service-set.h"
+#include "gnunet-service-set_union.h"
+#include "gnunet-service-set_intersection.h"
 #include "gnunet-service-set_protocol.h"
 #include "gnunet_statistics_service.h"
 
@@ -33,6 +35,35 @@
  */
 #define INCOMING_CHANNEL_TIMEOUT GNUNET_TIME_UNIT_MINUTES
 
+
+/**
+ * Lazy copy requests made by a client.
+ */
+struct LazyCopyRequest
+{
+  /**
+   * Kept in a DLL.
+   */
+  struct LazyCopyRequest *prev;
+
+  /**
+   * Kept in a DLL.
+   */
+  struct LazyCopyRequest *next;
+
+  /**
+   * Which set are we supposed to copy?
+   */
+  struct Set *source_set;
+
+  /**
+   * Cookie identifying the request.
+   */
+  uint32_t cookie;
+
+};
+
+
 /**
  * A listener is inhabited by a client, and waits for evaluation
  * requests from remote peers.
@@ -50,21 +81,24 @@ struct Listener
   struct Listener *prev;
 
   /**
-   * Client that owns the listener.
-   * Only one client may own a listener.
+   * Head of DLL of operations this listener is responsible for.
+   * Once the client has accepted/declined the operation, the
+   * operation is moved to the respective set's operation DLLS.
    */
-  struct GNUNET_SERVICE_Client *client;
+  struct Operation *op_head;
 
   /**
-   * Message queue for the client
+   * Tail of DLL of operations this listener is responsible for.
+   * Once the client has accepted/declined the operation, the
+   * operation is moved to the respective set's operation DLLS.
    */
-  struct GNUNET_MQ_Handle *client_mq;
+  struct Operation *op_tail;
 
   /**
-   * Application ID for the operation, used to distinguish
-   * multiple operations of the same type with the same peer.
+   * Client that owns the listener.
+   * Only one client may own a listener.
    */
-  struct GNUNET_HashCode app_id;
+  struct ClientState *cs;
 
   /**
    * The port we are listening on with CADET.
@@ -72,27 +106,18 @@ struct Listener
   struct GNUNET_CADET_Port *open_port;
 
   /**
+   * Application ID for the operation, used to distinguish
+   * multiple operations of the same type with the same peer.
+   */
+  struct GNUNET_HashCode app_id;
+
+  /**
    * The type of the operation.
    */
   enum GNUNET_SET_OperationType operation;
 };
 
 
-struct LazyCopyRequest
-{
-  struct Set *source_set;
-  uint32_t cookie;
-
-  struct LazyCopyRequest *prev;
-  struct LazyCopyRequest *next;
-};
-
-
-/**
- * Configuration of our local peer.
- */
-static const struct GNUNET_CONFIGURATION_Handle *configuration;
-
 /**
  * Handle to the cadet service, used to listen for and connect to
  * remote peers.
@@ -100,96 +125,48 @@ static const struct GNUNET_CONFIGURATION_Handle *configuration;
 static struct GNUNET_CADET_Handle *cadet;
 
 /**
- * Sets are held in a doubly linked list.
+ * DLL of lazy copy requests by this client.
  */
-static struct Set *sets_head;
+static struct LazyCopyRequest *lazy_copy_head;
 
 /**
- * Sets are held in a doubly linked list.
+ * DLL of lazy copy requests by this client.
  */
-static struct Set *sets_tail;
+static struct LazyCopyRequest *lazy_copy_tail;
 
 /**
- * Listeners are held in a doubly linked list.
+ * Generator for unique cookie we set per lazy copy request.
  */
-static struct Listener *listeners_head;
+static uint32_t lazy_copy_cookie;
 
 /**
- * Listeners are held in a doubly linked list.
+ * Statistics handle.
  */
-static struct Listener *listeners_tail;
+struct GNUNET_STATISTICS_Handle *_GSS_statistics;
 
 /**
- * Incoming sockets from remote peers are held in a doubly linked
- * list.
+ * Listeners are held in a doubly linked list.
  */
-static struct Operation *incoming_head;
+static struct Listener *listener_head;
 
 /**
- * Incoming sockets from remote peers are held in a doubly linked
- * list.
+ * Listeners are held in a doubly linked list.
  */
-static struct Operation *incoming_tail;
-
-static struct LazyCopyRequest *lazy_copy_head;
-static struct LazyCopyRequest *lazy_copy_tail;
-
-static uint32_t lazy_copy_cookie = 1;
+static struct Listener *listener_tail;
 
 /**
  * Counter for allocating unique IDs for clients, used to identify
  * incoming operation requests from remote peers, that the client can
- * choose to accept or refuse.
+ * choose to accept or refuse.  0 must not be used (reserved for
+ * uninitialized).
  */
-static uint32_t suggest_id = 1;
-
-/**
- * Statistics handle.
- */
-struct GNUNET_STATISTICS_Handle *_GSS_statistics;
-
-
-/**
- * Get set that is owned by the given client, if any.
- *
- * @param client client to look for
- * @return set that the client owns, NULL if the client
- *         does not own a set
- */
-static struct Set *
-set_get (struct GNUNET_SERVICE_Client *client)
-{
-  struct Set *set;
-
-  for (set = sets_head; NULL != set; set = set->next)
-    if (set->client == client)
-      return set;
-  return NULL;
-}
-
-
-/**
- * Get the listener associated with the given client, if any.
- *
- * @param client the client
- * @return listener associated with the client, NULL
- *         if there isn't any
- */
-static struct Listener *
-listener_get (struct GNUNET_SERVICE_Client *client)
-{
-  struct Listener *listener;
-
-  for (listener = listeners_head; NULL != listener; listener = listener->next)
-    if (listener->client == client)
-      return listener;
-  return NULL;
-}
+static uint32_t suggest_id;
 
 
 /**
  * Get the incoming socket associated with the given id.
  *
+ * @param listener the listener to look in
  * @param id id to look for
  * @return the incoming socket associated with the id,
  *         or NULL if there is none
@@ -197,43 +174,49 @@ listener_get (struct GNUNET_SERVICE_Client *client)
 static struct Operation *
 get_incoming (uint32_t id)
 {
-  struct Operation *op;
-
-  for (op = incoming_head; NULL != op; op = op->next)
-    if (op->suggest_id == id)
-    {
-      GNUNET_assert (GNUNET_YES == op->is_incoming);
-      return op;
-    }
+  for (struct Listener *listener = listener_head;
+       NULL != listener;
+       listener = listener->next)
+  {
+    for (struct Operation *op = listener->op_head; NULL != op; op = op->next)
+      if (op->suggest_id == id)
+        return op;
+  }
   return NULL;
 }
 
 
 /**
- * Destroy a listener, free all resources associated with it.
+ * Destroy an incoming request from a remote peer
  *
- * @param listener listener to destroy
+ * @param op remote request to destroy
  */
 static void
-listener_destroy (struct Listener *listener)
+incoming_destroy (struct Operation *op)
 {
-  /* If the client is not dead yet, destroy it.
-   * The client's destroy callback will destroy the listener again. */
-  if (NULL != listener->client)
-  {
-    struct GNUNET_SERVICE_Client *client = listener->client;
+  struct Listener *listener;
+  struct GNUNET_CADET_Channel *channel;
 
-    listener->client = NULL;
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Disconnecting listener client\n");
-    GNUNET_SERVICE_client_drop (client);
-    return;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Destroying incoming operation %p\n",
+              op);
+  if (NULL != (listener = op->listener))
+  {
+    GNUNET_CONTAINER_DLL_remove (listener->op_head,
+                                 listener->op_tail,
+                                 op);
+    op->listener = NULL;
+  }
+  if (NULL != op->timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (op->timeout_task);
+    op->timeout_task = NULL;
+  }
+  if (NULL != (channel = op->channel))
+  {
+    op->channel = NULL;
+    GNUNET_CADET_channel_destroy (channel);
   }
-  GNUNET_CADET_close_port (listener->open_port);
-  GNUNET_CONTAINER_DLL_remove (listeners_head,
-                               listeners_tail,
-                               listener);
-  GNUNET_free (listener);
 }
 
 
@@ -303,12 +286,11 @@ garbage_collect_cb (void *cls,
 static void
 collect_generation_garbage (struct Set *set)
 {
-  struct Operation *op;
   struct GarbageContext gc;
 
   gc.min_op_generation = UINT_MAX;
   gc.max_op_generation = 0;
-  for (op = set->ops_head; NULL != op; op = op->next)
+  for (struct Operation *op = set->ops_head; NULL != op; op = op->next)
   {
     gc.min_op_generation = GNUNET_MIN (gc.min_op_generation,
                                        op->generation_created);
@@ -322,23 +304,36 @@ collect_generation_garbage (struct Set *set)
 }
 
 
+/**
+ * Is @a generation in the range of exclusions?
+ *
+ * @param generation generation to query
+ * @param excluded array of generations where the element is excluded
+ * @param excluded_size length of the @a excluded array
+ * @return #GNUNET_YES if @a generation is in any of the ranges
+ */
 static int
 is_excluded_generation (unsigned int generation,
                         struct GenerationRange *excluded,
                         unsigned int excluded_size)
 {
-  unsigned int i;
-
-  for (i = 0; i < excluded_size; i++)
-  {
-    if ( (generation >= excluded[i].start) && (generation < excluded[i].end) )
+  for (unsigned int i = 0; i < excluded_size; i++)
+    if ( (generation >= excluded[i].start) &&
+         (generation < excluded[i].end) )
       return GNUNET_YES;
-  }
-
   return GNUNET_NO;
 }
 
 
+/**
+ * Is element @a ee part of the set during @a query_generation?
+ *
+ * @param ee element to test
+ * @param query_generation generation to query
+ * @param excluded array of generations where the element is excluded
+ * @param excluded_size length of the @a excluded array
+ * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
+ */
 static int
 is_element_of_generation (struct ElementEntry *ee,
                           unsigned int query_generation,
@@ -347,11 +342,12 @@ is_element_of_generation (struct ElementEntry *ee,
 {
   struct MutationEvent *mut;
   int is_present;
-  unsigned int i;
 
   GNUNET_assert (NULL != ee->mutations);
-
-  if (GNUNET_YES == is_excluded_generation (query_generation, excluded, excluded_size))
+  if (GNUNET_YES ==
+      is_excluded_generation (query_generation,
+                              excluded,
+                              excluded_size))
   {
     GNUNET_break (0);
     return GNUNET_NO;
@@ -361,7 +357,7 @@ is_element_of_generation (struct ElementEntry *ee,
 
   /* Could be made faster with binary search, but lists
      are small, so why bother. */
-  for (i = 0; i < ee->mutations_size; i++)
+  for (unsigned int i = 0; i < ee->mutations_size; i++)
   {
     mut = &ee->mutations[i];
 
@@ -373,7 +369,10 @@ is_element_of_generation (struct ElementEntry *ee,
       continue;
     }
 
-    if (GNUNET_YES == is_excluded_generation (mut->generation, excluded, excluded_size))
+    if (GNUNET_YES ==
+        is_excluded_generation (mut->generation,
+                                excluded,
+                                excluded_size))
     {
       /* The generation is excluded (because it belongs to another
          fork via a lazy copy) and thus mutations aren't considered
@@ -382,11 +381,12 @@ is_element_of_generation (struct ElementEntry *ee,
     }
 
     /* This would be an inconsistency in how we manage mutations. */
-    if ( (GNUNET_YES == is_present) && (GNUNET_YES == mut->added) )
+    if ( (GNUNET_YES == is_present) &&
+         (GNUNET_YES == mut->added) )
       GNUNET_assert (0);
-
     /* Likewise. */
-    if ( (GNUNET_NO == is_present) && (GNUNET_NO == mut->added) )
+    if ( (GNUNET_NO == is_present) &&
+         (GNUNET_NO == mut->added) )
       GNUNET_assert (0);
 
     is_present = mut->added;
@@ -396,44 +396,33 @@ is_element_of_generation (struct ElementEntry *ee,
 }
 
 
-int
-_GSS_is_element_of_set (struct ElementEntry *ee,
-                        struct Set *set)
-{
-  return is_element_of_generation (ee,
-                                   set->current_generation,
-                                   set->excluded_generations,
-                                   set->excluded_generations_size);
-}
-
-
-static int
-is_element_of_iteration (struct ElementEntry *ee,
-                         struct Set *set)
-{
-  return is_element_of_generation (ee,
-                                   set->iter_generation,
-                                   set->excluded_generations,
-                                   set->excluded_generations_size);
-}
-
-
+/**
+ * Is element @a ee part of the set used by @a op?
+ *
+ * @param ee element to test
+ * @param op operation the defines the set and its generation
+ * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
+ */
 int
 _GSS_is_element_of_operation (struct ElementEntry *ee,
                               struct Operation *op)
 {
   return is_element_of_generation (ee,
                                    op->generation_created,
-                                   op->spec->set->excluded_generations,
-                                   op->spec->set->excluded_generations_size);
+                                   op->set->excluded_generations,
+                                   op->set->excluded_generations_size);
 }
 
 
 /**
- * Destroy the given operation.  Call the implementation-specific
- * cancel function of the operation.  Disconnects from the remote
- * peer.  Does not disconnect the client, as there may be multiple
- * operations per set.
+ * Destroy the given operation.  Used for any operation where both
+ * peers were known and that thus actually had a vt and channel.  Must
+ * not be used for operations where 'listener' is still set and we do
+ * not know the other peer.
+ *
+ * Call the implementation-specific cancel function of the operation.
+ * Disconnects from the remote peer.  Does not disconnect the client,
+ * as there may be multiple operations per set.
  *
  * @param op operation to destroy
  * @param gc #GNUNET_YES to perform garbage collection on the set
@@ -442,38 +431,39 @@ void
 _GSS_operation_destroy (struct Operation *op,
                         int gc)
 {
-  struct Set *set;
+  struct Set *set = op->set;
   struct GNUNET_CADET_Channel *channel;
 
-  if (NULL == op->vt)
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Destroying operation %p\n",
+              op);
+  GNUNET_assert (NULL == op->listener);
+  if (NULL != op->state)
   {
-    /* already in #_GSS_operation_destroy() */
-    return;
+    set->vt->cancel (op);
+    op->state = NULL;
   }
-  GNUNET_assert (GNUNET_NO == op->is_incoming);
-  GNUNET_assert (NULL != op->spec);
-  set = op->spec->set;
-  GNUNET_CONTAINER_DLL_remove (set->ops_head,
-                               set->ops_tail,
-                               op);
-  op->vt->cancel (op);
-  op->vt = NULL;
-  if (NULL != op->spec)
+  if (NULL != set)
   {
-    if (NULL != op->spec->context_msg)
-    {
-      GNUNET_free (op->spec->context_msg);
-      op->spec->context_msg = NULL;
-    }
-    GNUNET_free (op->spec);
-    op->spec = NULL;
+    GNUNET_CONTAINER_DLL_remove (set->ops_head,
+                                 set->ops_tail,
+                                 op);
+    op->set = NULL;
+  }
+  if (NULL != op->context_msg)
+  {
+    GNUNET_free (op->context_msg);
+    op->context_msg = NULL;
   }
   if (NULL != (channel = op->channel))
   {
+    /* This will free op; called conditionally as this helper function
+       is also called from within the channel disconnect handler. */
     op->channel = NULL;
     GNUNET_CADET_channel_destroy (channel);
   }
-  if (GNUNET_YES == gc)
+  if ( (NULL != set) &&
+       (GNUNET_YES == gc) )
     collect_generation_garbage (set);
   /* We rely on the channel end handler to free 'op'. When 'op->channel' was NULL,
    * there was a channel end handler that will free 'op' on the call stack. */
@@ -481,6 +471,28 @@ _GSS_operation_destroy (struct Operation *op,
 
 
 /**
+ * Callback called when a client connects to the service.
+ *
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a `struct ClientState`
+ */
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
+{
+  struct ClientState *cs;
+
+  cs = GNUNET_new (struct ClientState);
+  cs->client = c;
+  cs->mq = mq;
+  return cs;
+}
+
+
+/**
  * Iterator over hash map entries to free element entries.
  *
  * @param cls closure
@@ -496,66 +508,76 @@ destroy_elements_iterator (void *cls,
   struct ElementEntry *ee = value;
 
   GNUNET_free_non_null (ee->mutations);
-
   GNUNET_free (ee);
   return GNUNET_YES;
 }
 
 
 /**
- * Destroy a set, and free all resources and operations associated with it.
+ * Clean up after a client has disconnected
  *
- * @param set the set to destroy
+ * @param cls closure, unused
+ * @param client the client to clean up after
+ * @param internal_cls the `struct ClientState`
  */
 static void
-set_destroy (struct Set *set)
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *client,
+                      void *internal_cls)
 {
-  if (NULL != set->client)
-  {
-    /* If the client is not dead yet, destroy it.  The client's destroy
-     * callback will call `set_destroy()` again in this case.  We do
-     * this so that the channel end handler still has a valid set handle
-     * to destroy. */
-    struct GNUNET_SERVICE_Client *client = set->client;
-
-    set->client = NULL;
-    GNUNET_SERVICE_client_drop (client);
-    return;
-  }
-  GNUNET_assert (NULL != set->state);
-  while (NULL != set->ops_head)
-    _GSS_operation_destroy (set->ops_head, GNUNET_NO);
-  set->vt->destroy_set (set->state);
-  set->state = NULL;
-  if (NULL != set->iter)
-  {
-    GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
-    set->iter = NULL;
-    set->iteration_id++;
-  }
+  struct ClientState *cs = internal_cls;
+  struct Operation *op;
+  struct Listener *listener;
+  struct Set *set;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Client disconnected, cleaning up\n");
+  if (NULL != (set = cs->set))
   {
-    struct SetContent *content;
+    struct SetContent *content = set->content;
     struct PendingMutation *pm;
     struct PendingMutation *pm_current;
+    struct LazyCopyRequest *lcr;
 
-    content = set->content;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Destroying client's set\n");
+    /* Destroy pending set operations */
+    while (NULL != set->ops_head)
+      _GSS_operation_destroy (set->ops_head,
+                              GNUNET_NO);
+
+    /* Destroy operation-specific state */
+    GNUNET_assert (NULL != set->state);
+    set->vt->destroy_set (set->state);
+    set->state = NULL;
+
+    /* Clean up ongoing iterations */
+    if (NULL != set->iter)
+    {
+      GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
+      set->iter = NULL;
+      set->iteration_id++;
+    }
 
-    // discard any pending mutations that reference this set
+    /* discard any pending mutations that reference this set */
     pm = content->pending_mutations_head;
     while (NULL != pm)
     {
       pm_current = pm;
       pm = pm->next;
-      if (pm_current-> set == set)
+      if (pm_current->set == set)
+      {
         GNUNET_CONTAINER_DLL_remove (content->pending_mutations_head,
                                      content->pending_mutations_tail,
                                      pm_current);
-
+        GNUNET_free (pm_current);
+      }
     }
 
+    /* free set content (or at least decrement RC) */
     set->content = NULL;
     GNUNET_assert (0 != content->refcount);
-    content->refcount -= 1;
+    content->refcount--;
     if (0 == content->refcount)
     {
       GNUNET_assert (NULL != content->elements);
@@ -566,154 +588,85 @@ set_destroy (struct Set *set)
       content->elements = NULL;
       GNUNET_free (content);
     }
-  }
-  GNUNET_free_non_null (set->excluded_generations);
-  set->excluded_generations = NULL;
-  GNUNET_CONTAINER_DLL_remove (sets_head,
-                               sets_tail,
-                               set);
+    GNUNET_free_non_null (set->excluded_generations);
+    set->excluded_generations = NULL;
 
-  // remove set from pending copy requests
-  {
-    struct LazyCopyRequest *lcr;
+    /* remove set from pending copy requests */
     lcr = lazy_copy_head;
     while (NULL != lcr)
     {
-      struct LazyCopyRequest *lcr_current;
-      lcr_current = lcr;
+      struct LazyCopyRequest *lcr_current = lcr;
+
       lcr = lcr->next;
       if (lcr_current->source_set == set)
+      {
         GNUNET_CONTAINER_DLL_remove (lazy_copy_head,
                                      lazy_copy_tail,
                                      lcr_current);
+        GNUNET_free (lcr_current);
+      }
     }
+    GNUNET_free (set);
   }
 
-  GNUNET_free (set);
-}
-
-
-/**
- * Callback called when a client connects to the service.
- *
- * @param cls closure for the service
- * @param c the new client that connected to the service
- * @param mq the message queue used to send messages to the client
- * @return @a c
- */
-static void *
-client_connect_cb (void *cls,
-                   struct GNUNET_SERVICE_Client *c,
-                   struct GNUNET_MQ_Handle *mq)
-{
-  return c;
-}
-
-
-/**
- * Clean up after a client has disconnected
- *
- * @param cls closure, unused
- * @param client the client to clean up after
- * @param internal_cls our client-specific internal data structure
- */
-static void
-client_disconnect_cb (void *cls,
-                      struct GNUNET_SERVICE_Client *client,
-                      void *internal_cls)
-{
-  struct Listener *listener;
-  struct Set *set;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "client disconnected, cleaning up\n");
-  set = set_get (client);
-  if (NULL != set)
+  if (NULL != (listener = cs->listener))
   {
-    set->client = NULL;
-    set_destroy (set);
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Client's set destroyed\n");
-  }
-  listener = listener_get (client);
-  if (NULL != listener)
-  {
-    listener->client = NULL;
-    listener_destroy (listener);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Client's listener destroyed\n");
+                "Destroying client's listener\n");
+    GNUNET_CADET_close_port (listener->open_port);
+    listener->open_port = NULL;
+    while (NULL != (op = listener->op_head))
+      incoming_destroy (op);
+    GNUNET_CONTAINER_DLL_remove (listener_head,
+                                 listener_tail,
+                                 listener);
+    GNUNET_free (listener);
   }
+  GNUNET_free (cs);
 }
 
 
 /**
- * Destroy an incoming request from a remote peer
+ * Check a request for a set operation from another peer.
  *
- * @param incoming remote request to destroy
+ * @param cls the operation state
+ * @param msg the received message
+ * @return #GNUNET_OK if the channel should be kept alive,
+ *         #GNUNET_SYSERR to destroy the channel
  */
-static void
-incoming_destroy (struct Operation *incoming)
+static int
+check_incoming_msg (void *cls,
+                    const struct OperationRequestMessage *msg)
 {
-  struct GNUNET_CADET_Channel *channel;
+  struct Operation *op = cls;
+  struct Listener *listener = op->listener;
+  const struct GNUNET_MessageHeader *nested_context;
 
-  GNUNET_assert (GNUNET_YES == incoming->is_incoming);
-  GNUNET_CONTAINER_DLL_remove (incoming_head,
-                               incoming_tail,
-                               incoming);
-  if (NULL != incoming->timeout_task)
+  /* double operation request */
+  if (0 != op->suggest_id)
   {
-    GNUNET_SCHEDULER_cancel (incoming->timeout_task);
-    incoming->timeout_task = NULL;
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
-  /* make sure that the tunnel end handler will not destroy us again */
-  incoming->vt = NULL;
-  if (NULL != incoming->spec)
+  /* This should be equivalent to the previous condition, but can't hurt to check twice */
+  if (NULL == op->listener)
   {
-    GNUNET_free (incoming->spec);
-    incoming->spec = NULL;
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
   }
-  if (NULL != (channel = incoming->channel))
+  if (listener->operation != (enum GNUNET_SET_OperationType) ntohl (msg->operation))
   {
-    incoming->channel = NULL;
-    GNUNET_CADET_channel_destroy (channel);
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
-}
-
-
-/**
- * Suggest the given request to the listener. The listening client can
- * then accept or reject the remote request.
- *
- * @param incoming the incoming peer with the request to suggest
- * @param listener the listener to suggest the request to
- */
-static void
-incoming_suggest (struct Operation *incoming,
-                  struct Listener *listener)
-{
-  struct GNUNET_MQ_Envelope *mqm;
-  struct GNUNET_SET_RequestMessage *cmsg;
-
-  GNUNET_assert (GNUNET_YES == incoming->is_incoming);
-  GNUNET_assert (NULL != incoming->spec);
-  GNUNET_assert (0 == incoming->suggest_id);
-  incoming->suggest_id = suggest_id++;
-  if (0 == suggest_id)
-    suggest_id++;
-  GNUNET_assert (NULL != incoming->timeout_task);
-  GNUNET_SCHEDULER_cancel (incoming->timeout_task);
-  incoming->timeout_task = NULL;
-  mqm = GNUNET_MQ_msg_nested_mh (cmsg,
-                                 GNUNET_MESSAGE_TYPE_SET_REQUEST,
-                                 incoming->spec->context_msg);
-  GNUNET_assert (NULL != mqm);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Suggesting incoming request with accept id %u to listener\n",
-              incoming->suggest_id);
-  cmsg->accept_id = htonl (incoming->suggest_id);
-  cmsg->peer_id = incoming->spec->peer;
-  GNUNET_MQ_send (listener->client_mq,
-                  mqm);
+  nested_context = GNUNET_MQ_extract_nested_mh (msg);
+  if ( (NULL != nested_context) &&
+       (ntohs (nested_context->size) > GNUNET_SET_CONTEXT_MESSAGE_MAX_SIZE) )
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
 }
 
 
@@ -729,93 +682,85 @@ incoming_suggest (struct Operation *incoming,
  * our virtual table and subsequent msgs would be routed differently (as
  * we then know what type of operation this is).
  *
- * @param op the operation state
- * @param mh the received message
+ * @param cls the operation state
+ * @param msg the received message
  * @return #GNUNET_OK if the channel should be kept alive,
  *         #GNUNET_SYSERR to destroy the channel
  */
-static int
-handle_incoming_msg (struct Operation *op,
-                     const struct GNUNET_MessageHeader *mh)
+static void
+handle_incoming_msg (void *cls,
+                     const struct OperationRequestMessage *msg)
 {
-  const struct OperationRequestMessage *msg;
+  struct Operation *op = cls;
   struct Listener *listener = op->listener;
-  struct OperationSpecification *spec;
   const struct GNUNET_MessageHeader *nested_context;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_SET_RequestMessage *cmsg;
 
-  msg = (const struct OperationRequestMessage *) mh;
-  GNUNET_assert (GNUNET_YES == op->is_incoming);
-  if (GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST != ntohs (mh->type))
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  /* double operation request */
-  if (NULL != op->spec)
-  {
-    GNUNET_break_op (0);
-    return GNUNET_SYSERR;
-  }
-  spec = GNUNET_new (struct OperationSpecification);
   nested_context = GNUNET_MQ_extract_nested_mh (msg);
-  if ( (NULL != nested_context) &&
-       (ntohs (nested_context->size) > GNUNET_SET_CONTEXT_MESSAGE_MAX_SIZE) )
-  {
-    GNUNET_break_op (0);
-    GNUNET_free (spec);
-    return GNUNET_SYSERR;
-  }
   /* Make a copy of the nested_context (application-specific context
      information that is opaque to set) so we can pass it to the
      listener later on */
   if (NULL != nested_context)
-    spec->context_msg = GNUNET_copy_message (nested_context);
-  spec->operation = ntohl (msg->operation);
-  spec->app_id = listener->app_id;
-  spec->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
-                                         UINT32_MAX);
-  spec->peer = op->peer;
-  spec->remote_element_count = ntohl (msg->element_count);
-  op->spec = spec;
-
-  listener = op->listener;
+    op->context_msg = GNUNET_copy_message (nested_context);
+  op->remote_element_count = ntohl (msg->element_count);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received P2P operation request (op %u, port %s) for active listener\n",
               (uint32_t) ntohl (msg->operation),
-              GNUNET_h2s (&listener->app_id));
-  incoming_suggest (op,
-                    listener);
-  return GNUNET_OK;
+              GNUNET_h2s (&op->listener->app_id));
+  GNUNET_assert (0 == op->suggest_id);
+  if (0 == suggest_id)
+    suggest_id++;
+  op->suggest_id = suggest_id++;
+  GNUNET_assert (NULL != op->timeout_task);
+  GNUNET_SCHEDULER_cancel (op->timeout_task);
+  op->timeout_task = NULL;
+  env = GNUNET_MQ_msg_nested_mh (cmsg,
+                                 GNUNET_MESSAGE_TYPE_SET_REQUEST,
+                                 op->context_msg);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Suggesting incoming request with accept id %u to listener %p of client %p\n",
+              op->suggest_id,
+              listener,
+              listener->cs);
+  cmsg->accept_id = htonl (op->suggest_id);
+  cmsg->peer_id = op->peer;
+  GNUNET_MQ_send (listener->cs->mq,
+                  env);
+  /* NOTE: GNUNET_CADET_receive_done() will be called in
+     #handle_client_accept() */
 }
 
 
+/**
+ * Add an element to @a set as specified by @a msg
+ *
+ * @param set set to manipulate
+ * @param msg message specifying the change
+ */
 static void
 execute_add (struct Set *set,
-             const struct GNUNET_MessageHeader *m)
+             const struct GNUNET_SET_ElementMessage *msg)
 {
-  const struct GNUNET_SET_ElementMessage *msg;
   struct GNUNET_SET_Element el;
   struct ElementEntry *ee;
   struct GNUNET_HashCode hash;
 
-  GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_ADD == ntohs (m->type));
-
-  msg = (const struct GNUNET_SET_ElementMessage *) m;
-  el.size = ntohs (m->size) - sizeof *msg;
+  GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_ADD == ntohs (msg->header.type));
+  el.size = ntohs (msg->header.size) - sizeof (*msg);
   el.data = &msg[1];
   el.element_type = ntohs (msg->element_type);
-  GNUNET_SET_element_hash (&el, &hash);
-
+  GNUNET_SET_element_hash (&el,
+                           &hash);
   ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements,
                                           &hash);
-
   if (NULL == ee)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Client inserts element %s of size %u\n",
                 GNUNET_h2s (&hash),
                 el.size);
-    ee = GNUNET_malloc (el.size + sizeof *ee);
+    ee = GNUNET_malloc (el.size + sizeof (*ee));
     ee->element.size = el.size;
     GNUNET_memcpy (&ee[1],
             el.data,
@@ -832,7 +777,11 @@ execute_add (struct Set *set,
                                                      ee,
                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   }
-  else if (GNUNET_YES == _GSS_is_element_of_set (ee, set))
+  else if (GNUNET_YES ==
+           is_element_of_generation (ee,
+                                     set->current_generation,
+                                     set->excluded_generations,
+                                     set->excluded_generations_size))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Client inserted element %s of size %u twice (ignored)\n",
@@ -852,24 +801,27 @@ execute_add (struct Set *set,
                          ee->mutations_size,
                          mut);
   }
-
-  set->vt->add (set->state, ee);
+  set->vt->add (set->state,
+                ee);
 }
 
 
+/**
+ * Remove an element from @a set as specified by @a msg
+ *
+ * @param set set to manipulate
+ * @param msg message specifying the change
+ */
 static void
 execute_remove (struct Set *set,
-                const struct GNUNET_MessageHeader *m)
+                const struct GNUNET_SET_ElementMessage *msg)
 {
-  const struct GNUNET_SET_ElementMessage *msg;
   struct GNUNET_SET_Element el;
   struct ElementEntry *ee;
   struct GNUNET_HashCode hash;
 
-  GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_REMOVE == ntohs (m->type));
-
-  msg = (const struct GNUNET_SET_ElementMessage *) m;
-  el.size = ntohs (m->size) - sizeof *msg;
+  GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_REMOVE == ntohs (msg->header.type));
+  el.size = ntohs (msg->header.size) - sizeof (*msg);
   el.data = &msg[1];
   el.element_type = ntohs (msg->element_type);
   GNUNET_SET_element_hash (&el, &hash);
@@ -883,7 +835,11 @@ execute_remove (struct Set *set,
                 el.size);
     return;
   }
-  if (GNUNET_NO == _GSS_is_element_of_set (ee, set))
+  if (GNUNET_NO ==
+      is_element_of_generation (ee,
+                                set->current_generation,
+                                set->excluded_generations,
+                                set->excluded_generations_size))
   {
     /* Client tried to remove element twice */
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -906,22 +862,28 @@ execute_remove (struct Set *set,
                          ee->mutations_size,
                          mut);
   }
-  set->vt->remove (set->state, ee);
+  set->vt->remove (set->state,
+                   ee);
 }
 
 
-
+/**
+ * Perform a mutation on a set as specified by the @a msg
+ *
+ * @param set the set to mutate
+ * @param msg specification of what to change
+ */
 static void
 execute_mutation (struct Set *set,
-                  const struct GNUNET_MessageHeader *m)
+                  const struct GNUNET_SET_ElementMessage *msg)
 {
-  switch (ntohs (m->type))
+  switch (ntohs (msg->header.type))
   {
     case GNUNET_MESSAGE_TYPE_SET_ADD:
-      execute_add (set, m);
+      execute_add (set, msg);
       break;
     case GNUNET_MESSAGE_TYPE_SET_REMOVE:
-      execute_remove (set, m);
+      execute_remove (set, msg);
       break;
     default:
       GNUNET_break (0);
@@ -929,6 +891,34 @@ execute_mutation (struct Set *set,
 }
 
 
+/**
+ * Execute mutations that were delayed on a set because of
+ * pending operations.
+ *
+ * @param set the set to execute mutations on
+ */
+static void
+execute_delayed_mutations (struct Set *set)
+{
+  struct PendingMutation *pm;
+
+  if (0 != set->content->iterator_count)
+    return; /* still cannot do this */
+  while (NULL != (pm = set->content->pending_mutations_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (set->content->pending_mutations_head,
+                                 set->content->pending_mutations_tail,
+                                 pm);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Executing pending mutation on %p.\n",
+                pm->set);
+    execute_mutation (pm->set,
+                      pm->msg);
+    GNUNET_free (pm->msg);
+    GNUNET_free (pm);
+  }
+}
+
 
 /**
  * Send the next element of a set to the set's client.  The next element is given by
@@ -952,65 +942,45 @@ send_client_element (struct Set *set)
   struct GNUNET_SET_IterResponseMessage *msg;
 
   GNUNET_assert (NULL != set->iter);
-
-again:
-
-  ret = GNUNET_CONTAINER_multihashmap_iterator_next (set->iter,
-                                                     NULL,
-                                                     (const void **) &ee);
-  if (GNUNET_NO == ret)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Iteration on %p done.\n",
-                (void *) set);
-    ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_ITER_DONE);
-    GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
-    set->iter = NULL;
-    set->iteration_id++;
-
-    GNUNET_assert (set->content->iterator_count > 0);
-    set->content->iterator_count -= 1;
-
-    if (0 == set->content->iterator_count)
+  do {
+    ret = GNUNET_CONTAINER_multihashmap_iterator_next (set->iter,
+                                                       NULL,
+                                                       (const void **) &ee);
+    if (GNUNET_NO == ret)
     {
-      while (NULL != set->content->pending_mutations_head)
-      {
-        struct PendingMutation *pm;
-
-        pm = set->content->pending_mutations_head;
-        GNUNET_CONTAINER_DLL_remove (set->content->pending_mutations_head,
-                                     set->content->pending_mutations_tail,
-                                     pm);
-        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                    "Executing pending mutation on %p.\n",
-                    (void *) pm->set);
-        execute_mutation (pm->set, pm->mutation_message);
-        GNUNET_free (pm->mutation_message);
-        GNUNET_free (pm);
-      }
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Iteration on %p done.\n",
+                  set);
+      ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_ITER_DONE);
+      GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
+      set->iter = NULL;
+      set->iteration_id++;
+      GNUNET_assert (set->content->iterator_count > 0);
+      set->content->iterator_count--;
+      execute_delayed_mutations (set);
+      GNUNET_MQ_send (set->cs->mq,
+                      ev);
+      return;
     }
-
-  }
-  else
-  {
     GNUNET_assert (NULL != ee);
-
-    if (GNUNET_NO == is_element_of_iteration (ee, set))
-      goto again;
-
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Sending iteration element on %p.\n",
-                (void *) set);
-    ev = GNUNET_MQ_msg_extra (msg,
-                              ee->element.size,
-                              GNUNET_MESSAGE_TYPE_SET_ITER_ELEMENT);
-    GNUNET_memcpy (&msg[1],
-            ee->element.data,
-            ee->element.size);
-    msg->element_type = htons (ee->element.element_type);
-    msg->iteration_id = htons (set->iteration_id);
-  }
-  GNUNET_MQ_send (set->client_mq, ev);
+  } while (GNUNET_NO ==
+           is_element_of_generation (ee,
+                                     set->iter_generation,
+                                     set->excluded_generations,
+                                     set->excluded_generations_size));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Sending iteration element on %p.\n",
+              set);
+  ev = GNUNET_MQ_msg_extra (msg,
+                            ee->element.size,
+                            GNUNET_MESSAGE_TYPE_SET_ITER_ELEMENT);
+  GNUNET_memcpy (&msg[1],
+                 ee->element.data,
+                 ee->element.size);
+  msg->element_type = htons (ee->element.element_type);
+  msg->iteration_id = htons (set->iteration_id);
+  GNUNET_MQ_send (set->cs->mq,
+                  ev);
 }
 
 
@@ -1027,22 +997,21 @@ static void
 handle_client_iterate (void *cls,
                        const struct GNUNET_MessageHeader *m)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
+  struct ClientState *cs = cls;
   struct Set *set;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* attempt to iterate over a non existing set */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   if (NULL != set->iter)
   {
     /* Only one concurrent iterate-action allowed per set */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1050,8 +1019,8 @@ handle_client_iterate (void *cls,
               (void *) set,
               set->current_generation,
               GNUNET_CONTAINER_multihashmap_size (set->content->elements));
-  GNUNET_SERVICE_client_continue (client);
-  set->content->iterator_count += 1;
+  GNUNET_SERVICE_client_continue (cs->client);
+  set->content->iterator_count++;
   set->iter = GNUNET_CONTAINER_multihashmap_iterator_create (set->content->elements);
   set->iter_generation = set->current_generation;
   send_client_element (set);
@@ -1070,17 +1039,17 @@ static void
 handle_client_create_set (void *cls,
                           const struct GNUNET_SET_CreateMessage *msg)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
+  struct ClientState *cs = cls;
   struct Set *set;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Client created new set (operation %u)\n",
               (uint32_t) ntohl (msg->operation));
-  if (NULL != set_get (client))
+  if (NULL != cs->set)
   {
     /* There can only be one set per client */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   set = GNUNET_new (struct Set);
@@ -1095,27 +1064,25 @@ handle_client_create_set (void *cls,
   default:
     GNUNET_free (set);
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-  set->operation = ntohl (msg->operation);
+  set->operation = (enum GNUNET_SET_OperationType) ntohl (msg->operation);
   set->state = set->vt->create ();
   if (NULL == set->state)
   {
     /* initialization failed (i.e. out of memory) */
     GNUNET_free (set);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   set->content = GNUNET_new (struct SetContent);
   set->content->refcount = 1;
-  set->content->elements = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_YES);
-  set->client = client;
-  set->client_mq = GNUNET_SERVICE_client_get_mq (client);
-  GNUNET_CONTAINER_DLL_insert (sets_head,
-                               sets_tail,
-                               set);
-  GNUNET_SERVICE_client_continue (client);
+  set->content->elements = GNUNET_CONTAINER_multihashmap_create (1,
+                                                                 GNUNET_YES);
+  set->cs = cs;
+  cs->set = set;
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1131,31 +1098,12 @@ handle_client_create_set (void *cls,
 static void
 incoming_timeout_cb (void *cls)
 {
-  struct Operation *incoming = cls;
+  struct Operation *op = cls;
 
-  incoming->timeout_task = NULL;
-  GNUNET_assert (GNUNET_YES == incoming->is_incoming);
+  op->timeout_task = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Remote peer's incoming request timed out\n");
-  incoming_destroy (incoming);
-}
-
-
-/**
- * Terminates an incoming operation in case we have not yet received an
- * operation request. Called by the channel destruction handler.
- *
- * @param op the channel context
- */
-static void
-handle_incoming_disconnect (struct Operation *op)
-{
-  GNUNET_assert (GNUNET_YES == op->is_incoming);
-  /* channel is already dead, incoming_destroy must not
-   * destroy it ... */
-  op->channel = NULL;
   incoming_destroy (op);
-  op->vt = NULL;
 }
 
 
@@ -1180,32 +1128,26 @@ channel_new_cb (void *cls,
                 struct GNUNET_CADET_Channel *channel,
                 const struct GNUNET_PeerIdentity *source)
 {
-  static const struct SetVT incoming_vt = {
-    .msg_handler = &handle_incoming_msg,
-    .peer_disconnect = &handle_incoming_disconnect
-  };
   struct Listener *listener = cls;
-  struct Operation *incoming;
+  struct Operation *op;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "New incoming channel\n");
-  incoming = GNUNET_new (struct Operation);
-  incoming->listener = listener;
-  incoming->is_incoming = GNUNET_YES;
-  incoming->peer = *source;
-  incoming->channel = channel;
-  incoming->mq = GNUNET_CADET_get_mq (incoming->channel);
-  incoming->vt = &incoming_vt;
-  incoming->timeout_task
+  op = GNUNET_new (struct Operation);
+  op->listener = listener;
+  op->peer = *source;
+  op->channel = channel;
+  op->mq = GNUNET_CADET_get_mq (op->channel);
+  op->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
+                                       UINT32_MAX);
+  op->timeout_task
     = GNUNET_SCHEDULER_add_delayed (INCOMING_CHANNEL_TIMEOUT,
                                     &incoming_timeout_cb,
-                                    incoming);
-  GNUNET_CONTAINER_DLL_insert_tail (incoming_head,
-                                    incoming_tail,
-                                    incoming);
-  // incoming_suggest (incoming,
-  //                  listener);
-  return incoming;
+                                    op);
+  GNUNET_CONTAINER_DLL_insert (listener->op_head,
+                               listener->op_tail,
+                               op);
+  return op;
 }
 
 
@@ -1234,22 +1176,14 @@ channel_end_cb (void *channel_ctx,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "channel_end_cb called\n");
   op->channel = NULL;
-  op->keep++;
-  /* the vt can be null if a client already requested canceling op. */
-  if (NULL != op->vt)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "calling peer disconnect due to channel end\n");
-    op->vt->peer_disconnect (op);
-  }
-  op->keep--;
-  if (0 == op->keep)
-  {
-    /* cadet will never call us with the context again! */
-    GNUNET_free (op);
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "channel_end_cb finished\n");
+  if (NULL != op->listener)
+    incoming_destroy (op);
+  else if (NULL != op->set)
+    op->set->vt->channel_death (op);
+  else
+    _GSS_operation_destroy (op,
+                            GNUNET_YES);
+  GNUNET_free (op);
 }
 
 
@@ -1275,60 +1209,6 @@ channel_window_cb (void *cls,
   /* FIXME: not implemented, we could do flow control here... */
 }
 
-/**
- * FIXME: hack-job. Migrate to proper handler array use!
- *
- * @param cls local state associated with the channel.
- * @param message The actual message.
- */
-static int
-check_p2p_message (void *cls,
-                   const struct GNUNET_MessageHeader *message)
-{
-  return GNUNET_OK;
-}
-
-
-/**
- * FIXME: hack-job. Migrate to proper handler array use!
- *
- * Functions with this signature are called whenever a message is
- * received via a cadet channel.
- *
- * The msg_handler is a virtual table set in initially either when a peer
- * creates a new channel with us, or once we create a new channel
- * ourselves (evaluate).
- *
- * Once we know the exact type of operation (union/intersection), the vt is
- * replaced with an operation specific instance (_GSS_[op]_vt).
- *
- * @param cls local state associated with the channel.
- * @param message The actual message.
- */
-static void
-handle_p2p_message (void *cls,
-                    const struct GNUNET_MessageHeader *message)
-{
-  struct Operation *op = cls;
-  int ret;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Dispatching cadet message (type: %u)\n",
-              ntohs (message->type));
-  /* do this before the handler, as the handler might kill the channel */
-  GNUNET_CADET_receive_done (op->channel);
-  if (NULL != op->vt)
-    ret = op->vt->msg_handler (op,
-                               message);
-  else
-    ret = GNUNET_SYSERR;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Handled cadet message (type: %u)\n",
-              ntohs (message->type));
-  if (GNUNET_OK != ret)
-    GNUNET_CADET_channel_destroy (op->channel);
-}
-
 
 /**
  * Called when a client wants to create a new listener.
@@ -1340,116 +1220,99 @@ static void
 handle_client_listen (void *cls,
                       const struct GNUNET_SET_ListenMessage *msg)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
+  struct ClientState *cs = cls;
   struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (incoming_msg,
                            GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
-                           struct GNUNET_MessageHeader,
+                           struct OperationRequestMessage,
                            NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_ibf,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF,
-                           struct GNUNET_MessageHeader,
+                           struct IBFMessage,
                            NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_elements,
                            GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS,
-                           struct GNUNET_MessageHeader,
+                           struct GNUNET_SET_ElementMessage,
                            NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_offer,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER,
                            struct GNUNET_MessageHeader,
                            NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_inquiry,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY,
-                           struct GNUNET_MessageHeader,
+                           struct InquiryMessage,
                            NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_demand,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND,
                            struct GNUNET_MessageHeader,
                            NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
-                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE,
-                           struct GNUNET_MessageHeader,
-                           NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
-                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE,
-                           struct GNUNET_MessageHeader,
-                           NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
-                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL,
-                           struct GNUNET_MessageHeader,
-                           NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_fixed_size (union_p2p_done,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (union_p2p_full_done,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_fixed_size (union_p2p_request_full,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL,
+                             struct GNUNET_MessageHeader,
+                             NULL),
+    GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE,
-                           struct GNUNET_MessageHeader,
+                           struct StrataEstimatorMessage,
                            NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC,
-                           struct GNUNET_MessageHeader,
+                           struct StrataEstimatorMessage,
                            NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_full_element,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT,
-                           struct GNUNET_MessageHeader,
-                           NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
-                           GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO,
-                           struct GNUNET_MessageHeader,
+                           struct GNUNET_SET_ElementMessage,
                            NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_fixed_size (intersection_p2p_element_info,
+                             GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO,
+                             struct IntersectionElementInfoMessage,
+                             NULL),
+    GNUNET_MQ_hd_var_size (intersection_p2p_bf,
                            GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF,
-                           struct GNUNET_MessageHeader,
-                           NULL),
-    GNUNET_MQ_hd_var_size (p2p_message,
-                           GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE,
-                           struct GNUNET_MessageHeader,
+                           struct BFMessage,
                            NULL),
+    GNUNET_MQ_hd_fixed_size (intersection_p2p_done,
+                             GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE,
+                             struct IntersectionDoneMessage,
+                             NULL),
     GNUNET_MQ_handler_end ()
   };
   struct Listener *listener;
 
-  if (NULL != listener_get (client))
+  if (NULL != cs->listener)
   {
     /* max. one active listener per client! */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   listener = GNUNET_new (struct Listener);
-  listener->client = client;
-  listener->client_mq = GNUNET_SERVICE_client_get_mq (client);
+  listener->cs = cs;
   listener->app_id = msg->app_id;
-  listener->operation = ntohl (msg->operation);
-  GNUNET_CONTAINER_DLL_insert_tail (listeners_head,
-                                    listeners_tail,
-                                    listener);
+  listener->operation = (enum GNUNET_SET_OperationType) ntohl (msg->operation);
+  GNUNET_CONTAINER_DLL_insert (listener_head,
+                               listener_tail,
+                               listener);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "New listener created (op %u, port %s)\n",
               listener->operation,
               GNUNET_h2s (&listener->app_id));
-  listener->open_port = GNUNET_CADET_open_porT (cadet,
-                                                &msg->app_id,
-                                                &channel_new_cb,
-                                                listener,
-                                                &channel_window_cb,
-                                                &channel_end_cb,
-                                                cadet_handlers);
-  /* check for existing incoming requests the listener might be interested in */
-  for (struct Operation *op = incoming_head; NULL != op; op = op->next)
-  {
-    if (NULL == op->spec)
-      continue; /* no details available yet */
-    if (0 != op->suggest_id)
-      continue; /* this one has been already suggested to a listener */
-    if (listener->operation != op->spec->operation)
-      continue; /* incompatible operation */
-    if (0 != GNUNET_CRYPTO_hash_cmp (&listener->app_id,
-                                     &op->spec->app_id))
-      continue; /* incompatible appliation */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Found matching existing request\n");
-    incoming_suggest (op,
-                      listener);
-  }
-  GNUNET_SERVICE_client_continue (client);
+  listener->open_port
+    = GNUNET_CADET_open_port (cadet,
+                              &msg->app_id,
+                              &channel_new_cb,
+                              listener,
+                              &channel_window_cb,
+                              &channel_end_cb,
+                              cadet_handlers);
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1464,23 +1327,26 @@ static void
 handle_client_reject (void *cls,
                       const struct GNUNET_SET_RejectMessage *msg)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
-  struct Operation *incoming;
+  struct ClientState *cs = cls;
+  struct Operation *op;
 
-  incoming = get_incoming (ntohl (msg->accept_reject_id));
-  if (NULL == incoming)
+  op = get_incoming (ntohl (msg->accept_reject_id));
+  if (NULL == op)
   {
-    /* no matching incoming operation for this reject */
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    /* no matching incoming operation for this reject;
+       could be that the other peer already disconnected... */
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Client rejected unknown operation %u\n",
+                (unsigned int) ntohl (msg->accept_reject_id));
+    GNUNET_SERVICE_client_continue (cs->client);
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Peer request (op %u, app %s) rejected by client\n",
-              incoming->spec->operation,
-              GNUNET_h2s (&incoming->spec->app_id));
-  GNUNET_CADET_channel_destroy (incoming->channel);
-  GNUNET_SERVICE_client_continue (client);
+              op->listener->operation,
+              GNUNET_h2s (&cs->listener->app_id));
+  GNUNET_CADET_channel_destroy (op->channel);
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1488,13 +1354,14 @@ handle_client_reject (void *cls,
  * Called when a client wants to add or remove an element to a set it inhabits.
  *
  * @param cls client that sent the message
- * @param m message sent by the client
+ * @param msg message sent by the client
  */
 static int
 check_client_mutation (void *cls,
-                       const struct GNUNET_MessageHeader *m)
+                       const struct GNUNET_SET_ElementMessage *msg)
 {
-  /* FIXME: any check we might want to do here? */
+  /* NOTE: Technically, we should probably check with the
+     block library whether the element we are given is well-formed */
   return GNUNET_OK;
 }
 
@@ -1503,25 +1370,23 @@ check_client_mutation (void *cls,
  * Called when a client wants to add or remove an element to a set it inhabits.
  *
  * @param cls client that sent the message
- * @param m message sent by the client
+ * @param msg message sent by the client
  */
 static void
 handle_client_mutation (void *cls,
-                        const struct GNUNET_MessageHeader *m)
+                        const struct GNUNET_SET_ElementMessage *msg)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
+  struct ClientState *cs = cls;
   struct Set *set;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* client without a set requested an operation */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-
-  GNUNET_SERVICE_client_continue (client);
+  GNUNET_SERVICE_client_continue (cs->client);
 
   if (0 != set->content->iterator_count)
   {
@@ -1529,16 +1394,18 @@ handle_client_mutation (void *cls,
 
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Scheduling mutation on set\n");
-
     pm = GNUNET_new (struct PendingMutation);
-    pm->mutation_message = GNUNET_copy_message (m);
+    pm->msg = (struct GNUNET_SET_ElementMessage *) GNUNET_copy_message (&msg->header);
     pm->set = set;
     GNUNET_CONTAINER_DLL_insert_tail (set->content->pending_mutations_head,
                                       set->content->pending_mutations_tail,
                                       pm);
     return;
   }
-  execute_mutation (set, m);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Executing mutation on set\n");
+  execute_mutation (set,
+                    msg);
 }
 
 
@@ -1555,8 +1422,8 @@ advance_generation (struct Set *set)
 
   if (set->current_generation == set->content->latest_generation)
   {
-    set->content->latest_generation += 1;
-    set->current_generation += 1;
+    set->content->latest_generation++;
+    set->current_generation++;
     return;
   }
 
@@ -1564,10 +1431,8 @@ advance_generation (struct Set *set)
 
   r.start = set->current_generation + 1;
   r.end = set->content->latest_generation + 1;
-
   set->content->latest_generation = r.end;
   set->current_generation = r.end;
-
   GNUNET_array_append (set->excluded_generations,
                        set->excluded_generations_size,
                        r);
@@ -1605,112 +1470,105 @@ static void
 handle_client_evaluate (void *cls,
                         const struct GNUNET_SET_EvaluateMessage *msg)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
+  struct ClientState *cs = cls;
   struct Operation *op = GNUNET_new (struct Operation);
   const struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (incoming_msg,
                            GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
-                           struct GNUNET_MessageHeader,
+                           struct OperationRequestMessage,
                            op),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_ibf,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF,
-                           struct GNUNET_MessageHeader,
+                           struct IBFMessage,
                            op),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_elements,
                            GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS,
-                           struct GNUNET_MessageHeader,
+                           struct GNUNET_SET_ElementMessage,
                            op),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_offer,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER,
                            struct GNUNET_MessageHeader,
                            op),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_inquiry,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY,
-                           struct GNUNET_MessageHeader,
+                           struct InquiryMessage,
                            op),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_demand,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND,
                            struct GNUNET_MessageHeader,
                            op),
-    GNUNET_MQ_hd_var_size (p2p_message,
-                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE,
-                           struct GNUNET_MessageHeader,
-                           op),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_fixed_size (union_p2p_done,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE,
+                             struct GNUNET_MessageHeader,
+                             op),
+    GNUNET_MQ_hd_fixed_size (union_p2p_full_done,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE,
+                             struct GNUNET_MessageHeader,
+                             op),
+    GNUNET_MQ_hd_fixed_size (union_p2p_request_full,
+                             GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL,
+                             struct GNUNET_MessageHeader,
+                             op),
+    GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE,
-                           struct GNUNET_MessageHeader,
+                           struct StrataEstimatorMessage,
                            op),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC,
-                           struct GNUNET_MessageHeader,
-                           op),
-    GNUNET_MQ_hd_var_size (p2p_message,
-                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE,
-                           struct GNUNET_MessageHeader,
+                           struct StrataEstimatorMessage,
                            op),
-    GNUNET_MQ_hd_var_size (p2p_message,
-                           GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL,
-                           struct GNUNET_MessageHeader,
-                           op),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_var_size (union_p2p_full_element,
                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT,
-                           struct GNUNET_MessageHeader,
+                           struct GNUNET_SET_ElementMessage,
                            op),
-    GNUNET_MQ_hd_var_size (p2p_message,
-                           GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO,
-                           struct GNUNET_MessageHeader,
-                           op),
-    GNUNET_MQ_hd_var_size (p2p_message,
+    GNUNET_MQ_hd_fixed_size (intersection_p2p_element_info,
+                             GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO,
+                             struct IntersectionElementInfoMessage,
+                             op),
+    GNUNET_MQ_hd_var_size (intersection_p2p_bf,
                            GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF,
-                           struct GNUNET_MessageHeader,
-                           op),
-    GNUNET_MQ_hd_var_size (p2p_message,
-                           GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE,
-                           struct GNUNET_MessageHeader,
+                           struct BFMessage,
                            op),
+    GNUNET_MQ_hd_fixed_size (intersection_p2p_done,
+                             GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE,
+                             struct IntersectionDoneMessage,
+                             op),
     GNUNET_MQ_handler_end ()
   };
   struct Set *set;
-  struct OperationSpecification *spec;
   const struct GNUNET_MessageHeader *context;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     GNUNET_break (0);
     GNUNET_free (op);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-  spec = GNUNET_new (struct OperationSpecification);
-  spec->operation = set->operation;
-  spec->app_id = msg->app_id;
-  spec->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
-                                         UINT32_MAX);
-  spec->peer = msg->target_peer;
-  spec->set = set;
-  spec->result_mode = ntohl (msg->result_mode);
-  spec->client_request_id = ntohl (msg->request_id);
-  spec->byzantine = msg->byzantine;
-  spec->byzantine_lower_bound = msg->byzantine_lower_bound;
-  spec->force_full = msg->force_full;
-  spec->force_delta = msg->force_delta;
+  op->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
+                                       UINT32_MAX);
+  op->peer = msg->target_peer;
+  op->result_mode = ntohl (msg->result_mode);
+  op->client_request_id = ntohl (msg->request_id);
+  op->byzantine = msg->byzantine;
+  op->byzantine_lower_bound = msg->byzantine_lower_bound;
+  op->force_full = msg->force_full;
+  op->force_delta = msg->force_delta;
   context = GNUNET_MQ_extract_nested_mh (msg);
-  op->spec = spec;
 
-  // Advance generation values, so that
-  // mutations won't interfer with the running operation.
+  /* Advance generation values, so that
+     mutations won't interfer with the running operation. */
+  op->set = set;
   op->generation_created = set->current_generation;
   advance_generation (set);
-
-  op->vt = set->vt;
   GNUNET_CONTAINER_DLL_insert (set->ops_head,
                                set->ops_tail,
                                op);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Creating new CADET channel to port %s\n",
-              GNUNET_h2s (&msg->app_id));
-  op->channel = GNUNET_CADET_channel_creatE (cadet,
+              "Creating new CADET channel to port %s for set operation type %u\n",
+              GNUNET_h2s (&msg->app_id),
+              set->operation);
+  op->channel = GNUNET_CADET_channel_create (cadet,
                                              op,
                                              &msg->target_peer,
                                              &msg->app_id,
@@ -1719,9 +1577,15 @@ handle_client_evaluate (void *cls,
                                              &channel_end_cb,
                                              cadet_handlers);
   op->mq = GNUNET_CADET_get_mq (op->channel);
-  set->vt->evaluate (op,
-                     context);
-  GNUNET_SERVICE_client_continue (client);
+  op->state = set->vt->evaluate (op,
+                                 context);
+  if (NULL == op->state)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (cs->client);
+    return;
+  }
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1737,15 +1601,14 @@ static void
 handle_client_iter_ack (void *cls,
                         const struct GNUNET_SET_IterAckMessage *ack)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
+  struct ClientState *cs = cls;
   struct Set *set;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* client without a set acknowledged receiving a value */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   if (NULL == set->iter)
@@ -1753,10 +1616,10 @@ handle_client_iter_ack (void *cls,
     /* client sent an ack, but we were not expecting one (as
        set iteration has finished) */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-  GNUNET_SERVICE_client_continue (client);
+  GNUNET_SERVICE_client_continue (cs->client);
   if (ntohl (ack->send_more))
   {
     send_client_element (set);
@@ -1780,42 +1643,33 @@ static void
 handle_client_copy_lazy_prepare (void *cls,
                                  const struct GNUNET_MessageHeader *mh)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
+  struct ClientState *cs = cls;
   struct Set *set;
   struct LazyCopyRequest *cr;
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_SET_CopyLazyResponseMessage *resp_msg;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* client without a set requested an operation */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Client requested creation of lazy copy\n");
   cr = GNUNET_new (struct LazyCopyRequest);
-
-  cr->cookie = lazy_copy_cookie;
-  lazy_copy_cookie += 1;
+  cr->cookie = ++lazy_copy_cookie;
   cr->source_set = set;
-
   GNUNET_CONTAINER_DLL_insert (lazy_copy_head,
                                lazy_copy_tail,
                                cr);
-
-
   ev = GNUNET_MQ_msg (resp_msg,
                       GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_RESPONSE);
   resp_msg->cookie = cr->cookie;
-  GNUNET_MQ_send (set->client_mq, ev);
-
-
-  GNUNET_SERVICE_client_continue (client);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Client requested lazy copy\n");
+  GNUNET_MQ_send (set->cs->mq,
+                  ev);
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1829,21 +1683,19 @@ static void
 handle_client_copy_lazy_connect (void *cls,
                                  const struct GNUNET_SET_CopyLazyConnectMessage *msg)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
+  struct ClientState *cs = cls;
   struct LazyCopyRequest *cr;
   struct Set *set;
   int found;
 
-  if (NULL != set_get (client))
+  if (NULL != cs->set)
   {
     /* There can only be one set per client */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-
   found = GNUNET_NO;
-
   for (cr = lazy_copy_head; NULL != cr; cr = cr->next)
   {
     if (cr->cookie == msg->cookie)
@@ -1852,21 +1704,20 @@ handle_client_copy_lazy_connect (void *cls,
       break;
     }
   }
-
   if (GNUNET_NO == found)
   {
     /* client asked for copy with cookie we don't know */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-
   GNUNET_CONTAINER_DLL_remove (lazy_copy_head,
                                lazy_copy_tail,
                                cr);
-
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Client %p requested use of lazy copy\n",
+              cs);
   set = GNUNET_new (struct Set);
-
   switch (cr->source_set->operation)
   {
   case GNUNET_SET_OPERATION_INTERSECTION:
@@ -1886,37 +1737,28 @@ handle_client_copy_lazy_connect (void *cls,
     GNUNET_break (0);
     GNUNET_free (set);
     GNUNET_free (cr);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
 
   set->operation = cr->source_set->operation;
-  set->state = set->vt->copy_state (cr->source_set);
+  set->state = set->vt->copy_state (cr->source_set->state);
   set->content = cr->source_set->content;
-  set->content->refcount += 1;
+  set->content->refcount++;
 
   set->current_generation = cr->source_set->current_generation;
   set->excluded_generations_size = cr->source_set->excluded_generations_size;
-  set->excluded_generations = GNUNET_memdup (cr->source_set->excluded_generations,
-                                             set->excluded_generations_size * sizeof (struct GenerationRange));
+  set->excluded_generations
+    = GNUNET_memdup (cr->source_set->excluded_generations,
+                     set->excluded_generations_size * sizeof (struct GenerationRange));
 
   /* Advance the generation of the new set, so that mutations to the
      of the cloned set and the source set are independent. */
   advance_generation (set);
-
-
-  set->client = client;
-  set->client_mq = GNUNET_SERVICE_client_get_mq (client);
-  GNUNET_CONTAINER_DLL_insert (sets_head,
-                               sets_tail,
-                               set);
-
+  set->cs = cs;
+  cs->set = set;
   GNUNET_free (cr);
-
-  GNUNET_SERVICE_client_continue (client);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Client connected to lazy set\n");
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1930,26 +1772,22 @@ static void
 handle_client_cancel (void *cls,
                       const struct GNUNET_SET_CancelMessage *msg)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
+  struct ClientState *cs = cls;
   struct Set *set;
   struct Operation *op;
   int found;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* client without a set requested an operation */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Client requested cancel for op %u\n",
-              (uint32_t) ntohl (msg->request_id));
   found = GNUNET_NO;
   for (op = set->ops_head; NULL != op; op = op->next)
   {
-    if (op->spec->client_request_id == ntohl (msg->request_id))
+    if (op->client_request_id == ntohl (msg->request_id))
     {
       found = GNUNET_YES;
       break;
@@ -1962,15 +1800,19 @@ handle_client_cancel (void *cls,
      * yet and try to cancel the (just barely non-existent) operation.
      * So this is not a hard error.
      */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Client canceled non-existent op\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Client canceled non-existent op %u\n",
+                (uint32_t) ntohl (msg->request_id));
   }
   else
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Client requested cancel for op %u\n",
+                (uint32_t) ntohl (msg->request_id));
     _GSS_operation_destroy (op,
                             GNUNET_YES);
   }
-  GNUNET_SERVICE_client_continue (client);
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
@@ -1986,18 +1828,18 @@ static void
 handle_client_accept (void *cls,
                       const struct GNUNET_SET_AcceptMessage *msg)
 {
-  struct GNUNET_SERVICE_Client *client = cls;
+  struct ClientState *cs = cls;
   struct Set *set;
   struct Operation *op;
   struct GNUNET_SET_ResultMessage *result_message;
   struct GNUNET_MQ_Envelope *ev;
+  struct Listener *listener;
 
-  set = set_get (client);
-  if (NULL == set)
+  if (NULL == (set = cs->set))
   {
     /* client without a set requested to accept */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (client);
+    GNUNET_SERVICE_client_drop (cs->client);
     return;
   }
   op = get_incoming (ntohl (msg->accept_reject_id));
@@ -2005,71 +1847,75 @@ handle_client_accept (void *cls,
   {
     /* It is not an error if the set op does not exist -- it may
      * have been destroyed when the partner peer disconnected. */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Client accepted request that is no longer active\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Client %p accepted request %u of listener %p that is no longer active\n",
+                cs,
+                ntohl (msg->accept_reject_id),
+                cs->listener);
     ev = GNUNET_MQ_msg (result_message,
                         GNUNET_MESSAGE_TYPE_SET_RESULT);
     result_message->request_id = msg->request_id;
-    result_message->element_type = 0;
     result_message->result_status = htons (GNUNET_SET_STATUS_FAILURE);
-    GNUNET_MQ_send (set->client_mq, ev);
-    GNUNET_SERVICE_client_continue (client);
+    GNUNET_MQ_send (set->cs->mq,
+                    ev);
+    GNUNET_SERVICE_client_continue (cs->client);
     return;
   }
-
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Client accepting request %u\n",
               (uint32_t) ntohl (msg->accept_reject_id));
-  GNUNET_assert (GNUNET_YES == op->is_incoming);
-  op->is_incoming = GNUNET_NO;
-  GNUNET_CONTAINER_DLL_remove (incoming_head,
-                               incoming_tail,
+  listener = op->listener;
+  op->listener = NULL;
+  GNUNET_CONTAINER_DLL_remove (listener->op_head,
+                               listener->op_tail,
                                op);
-  op->spec->set = set;
+  op->set = set;
   GNUNET_CONTAINER_DLL_insert (set->ops_head,
                                set->ops_tail,
                                op);
-  op->spec->client_request_id = ntohl (msg->request_id);
-  op->spec->result_mode = ntohl (msg->result_mode);
-  op->spec->byzantine = msg->byzantine;
-  op->spec->byzantine_lower_bound = msg->byzantine_lower_bound;
-  op->spec->force_full = msg->force_full;
-  op->spec->force_delta = msg->force_delta;
-
-  // Advance generation values, so that
-  // mutations won't interfer with the running operation.
+  op->client_request_id = ntohl (msg->request_id);
+  op->result_mode = ntohl (msg->result_mode);
+  op->byzantine = msg->byzantine;
+  op->byzantine_lower_bound = msg->byzantine_lower_bound;
+  op->force_full = msg->force_full;
+  op->force_delta = msg->force_delta;
+
+  /* Advance generation values, so that future mutations do not
+     interfer with the running operation. */
   op->generation_created = set->current_generation;
   advance_generation (set);
-
-  op->vt = set->vt;
-  op->vt->accept (op);
-  GNUNET_SERVICE_client_continue (client);
+  GNUNET_assert (NULL == op->state);
+  op->state = set->vt->accept (op);
+  if (NULL == op->state)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (cs->client);
+    return;
+  }
+  /* Now allow CADET to continue, as we did not do this in
+     #handle_incoming_msg (as we wanted to first see if the
+     local client would accept the request). */
+  GNUNET_CADET_receive_done (op->channel);
+  GNUNET_SERVICE_client_continue (cs->client);
 }
 
 
 /**
  * Called to clean up, after a shutdown has been requested.
  *
- * @param cls closure
+ * @param cls closure, NULL
  */
 static void
 shutdown_task (void *cls)
 {
-  while (NULL != incoming_head)
-    incoming_destroy (incoming_head);
-  while (NULL != listeners_head)
-    listener_destroy (listeners_head);
-  while (NULL != sets_head)
-    set_destroy (sets_head);
-
-  /* it's important to destroy cadet at the end, as all channels
-   * must be destroyed before the cadet handle! */
+  /* Delay actual shutdown to allow service to disconnect clients */
   if (NULL != cadet)
   {
     GNUNET_CADET_disconnect (cadet);
     cadet = NULL;
   }
-  GNUNET_STATISTICS_destroy (_GSS_statistics, GNUNET_YES);
+  GNUNET_STATISTICS_destroy (_GSS_statistics,
+                             GNUNET_YES);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "handled shutdown request\n");
 }
@@ -2088,15 +1934,19 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *cfg,
      struct GNUNET_SERVICE_Handle *service)
 {
-  configuration = cfg;
+  /* FIXME: need to modify SERVICE (!) API to allow
+     us to run a shutdown task *after* clients were
+     forcefully disconnected! */
   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
                                  NULL);
-  _GSS_statistics = GNUNET_STATISTICS_create ("set", cfg);
-  cadet = GNUNET_CADET_connecT (cfg);
+  _GSS_statistics = GNUNET_STATISTICS_create ("set",
+                                              cfg);
+  cadet = GNUNET_CADET_connect (cfg);
   if (NULL == cadet)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 _("Could not connect to CADET service\n"));
+    GNUNET_SCHEDULER_shutdown ();
     return;
   }
 }
@@ -2122,7 +1972,7 @@ GNUNET_SERVICE_MAIN
                           NULL),
  GNUNET_MQ_hd_var_size (client_mutation,
                         GNUNET_MESSAGE_TYPE_SET_ADD,
-                        struct GNUNET_MessageHeader,
+                        struct GNUNET_SET_ElementMessage,
                         NULL),
  GNUNET_MQ_hd_fixed_size (client_create_set,
                           GNUNET_MESSAGE_TYPE_SET_CREATE,
@@ -2146,7 +1996,7 @@ GNUNET_SERVICE_MAIN
                           NULL),
  GNUNET_MQ_hd_var_size (client_mutation,
                         GNUNET_MESSAGE_TYPE_SET_REMOVE,
-                        struct GNUNET_MessageHeader,
+                        struct GNUNET_SET_ElementMessage,
                         NULL),
  GNUNET_MQ_hd_fixed_size (client_cancel,
                           GNUNET_MESSAGE_TYPE_SET_CANCEL,
diff --git a/src/set/gnunet-service-set.h b/src/set/gnunet-service-set.h
index 68d8fe81f..19413fd30 100644
--- a/src/set/gnunet-service-set.h
+++ b/src/set/gnunet-service-set.h
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2013, 2014 GNUnet e.V.
+      Copyright (C) 2013-2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -68,92 +68,13 @@ struct Operation;
 
 
 /**
- * Detail information about an operation.
- */
-struct OperationSpecification
-{
-
-  /**
-   * The remove peer we evaluate the operation with.
-   */
-  struct GNUNET_PeerIdentity peer;
-
-  /**
-   * Application ID for the operation, used to distinguish
-   * multiple operations of the same type with the same peer.
-   */
-  struct GNUNET_HashCode app_id;
-
-  /**
-   * Context message, may be NULL.
-   */
-  struct GNUNET_MessageHeader *context_msg;
-
-  /**
-   * Set associated with the operation, NULL until the spec has been
-   * associated with a set.
-   */
-  struct Set *set;
-
-  /**
-   * Salt to use for the operation.
-   */
-  uint32_t salt;
-
-  /**
-   * Remote peers element count
-   */
-  uint32_t remote_element_count;
-
-  /**
-   * ID used to identify an operation between service and client
-   */
-  uint32_t client_request_id;
-
-  /**
-   * The type of the operation.
-   */
-  enum GNUNET_SET_OperationType operation;
-
-  /**
-   * When are elements sent to the client, and which elements are sent?
-   */
-  enum GNUNET_SET_ResultMode result_mode;
-
-  /**
-   * Always use delta operation instead of sending full sets,
-   * even it it's less efficient.
-   */
-  int force_delta;
-
-  /**
-   * Always send full sets, even if delta operations would
-   * be more efficient.
-   */
-  int force_full;
-
-  /**
-   * #GNUNET_YES to fail operations where Byzantine faults
-   * are suspected
-   */
-  int byzantine;
-
-  /**
-   * Lower bound for the set size, used only when
-   * byzantine mode is enabled.
-   */
-  int byzantine_lower_bound;
-};
-
-
-/**
  * Signature of functions that create the implementation-specific
  * state for a set supporting a specific operation.
  *
  * @return a set state specific to the supported operation, NULL on error
  */
 typedef struct SetState *
-(*CreateImpl) (void);
+(*SetCreateImpl) (void);
 
 
 /**
@@ -164,18 +85,18 @@ typedef struct SetState *
  * @param ee element message from the client
  */
 typedef void
-(*AddRemoveImpl) (struct SetState *state,
+(*SetAddRemoveImpl) (struct SetState *state,
                   struct ElementEntry *ee);
 
 
 /**
- * Signature of functions that handle disconnection of the remote
- * peer.
+ * Make a copy of a set's internal state.
  *
- * @param op the set operation, contains implementation-specific data
+ * @param state set state to copy
+ * @return copy of the internal state
  */
-typedef void
-(*PeerDisconnectImpl) (struct Operation *op);
+typedef struct SetState *
+(*SetCopyStateImpl) (struct SetState *state);
 
 
 /**
@@ -185,7 +106,7 @@ typedef void
  * @param state the set state, contains implementation-specific data
  */
 typedef void
-(*DestroySetImpl) (struct SetState *state);
+(*SetDestroyImpl) (struct SetState *state);
 
 
 /**
@@ -193,8 +114,9 @@ typedef void
  *
  * @param op operation that is created by accepting the operation,
  *        should be initialized by the implementation
+ * @return operation-specific state to keep in @a op
  */
-typedef void
+typedef struct OperationState *
 (*OpAcceptImpl) (struct Operation *op);
 
 
@@ -206,38 +128,32 @@ typedef void
  *        begin the evaluation
  * @param opaque_context message to be transmitted to the listener
  *        to convince him to accept, may be NULL
+ * @return operation-specific state to keep in @a op
  */
-typedef void
+typedef struct OperationState *
 (*OpEvaluateImpl) (struct Operation *op,
                    const struct GNUNET_MessageHeader *opaque_context);
 
-
 /**
- * Signature of functions that implement the message handling for
- * the different set operations.
+ * Signature of functions that implement operation cancelation.
+ * This includes notifying the client about the operation's final
+ * state.
  *
  * @param op operation state
- * @param msg received message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR to
- *         destroy the operation and the tunnel
  */
-typedef int
-(*MsgHandlerImpl) (struct Operation *op,
-                   const struct GNUNET_MessageHeader *msg);
+typedef void
+(*OpCancelImpl) (struct Operation *op);
 
 
 /**
- * Signature of functions that implement operation cancellation
+ * Signature of functions called when the CADET channel died.
  *
  * @param op operation state
  */
 typedef void
-(*CancelImpl) (struct Operation *op);
+(*OpChannelDeathImpl) (struct Operation *op);
 
 
-typedef struct SetState *
-(*CopyStateImpl) (struct Set *op);
-
 
 /**
  * Dispatch table for a specific set operation.  Every set operation
@@ -248,49 +164,48 @@ struct SetVT
   /**
    * Callback for the set creation.
    */
-  CreateImpl create;
+  SetCreateImpl create;
 
   /**
    * Callback for element insertion
    */
-  AddRemoveImpl add;
+  SetAddRemoveImpl add;
 
   /**
    * Callback for element removal.
    */
-  AddRemoveImpl remove;
+  SetAddRemoveImpl remove;
 
   /**
-   * Callback for accepting a set operation request
+   * Callback for making a copy of a set's internal state.
    */
-  OpAcceptImpl accept;
+  SetCopyStateImpl copy_state;
 
   /**
-   * Callback for starting evaluation with a remote peer.
+   * Callback for destruction of the set state.
    */
-  OpEvaluateImpl evaluate;
+  SetDestroyImpl destroy_set;
 
   /**
-   * Callback for destruction of the set state.
+   * Callback for accepting a set operation request
    */
-  DestroySetImpl destroy_set;
+  OpAcceptImpl accept;
 
   /**
-   * Callback for handling operation-specific messages.
+   * Callback for starting evaluation with a remote peer.
    */
-  MsgHandlerImpl msg_handler;
+  OpEvaluateImpl evaluate;
 
   /**
-   * Callback for handling the remote peer's disconnect.
+   * Callback for canceling an operation.
    */
-  PeerDisconnectImpl peer_disconnect;
+  OpCancelImpl cancel;
 
   /**
-   * Callback for canceling an operation by its ID.
+   * Callback called in case the CADET channel died.
    */
-  CancelImpl cancel;
+  OpChannelDeathImpl channel_death;
 
-  CopyStateImpl copy_state;
 };
 
 
@@ -360,20 +275,56 @@ struct ElementEntry
 };
 
 
+/**
+ * A listener is inhabited by a client, and waits for evaluation
+ * requests from remote peers.
+ */
 struct Listener;
 
 
 /**
+ * State we keep per client.
+ */
+struct ClientState
+{
+  /**
+   * Set, if associated with the client, otherwise NULL.
+   */
+  struct Set *set;
+
+  /**
+   * Listener, if associated with the client, otherwise NULL.
+   */
+  struct Listener *listener;
+
+  /**
+   * Client handle.
+   */
+  struct GNUNET_SERVICE_Client *client;
+
+  /**
+   * Message queue.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+};
+
+
+/**
  * Operation context used to execute a set operation.
  */
 struct Operation
 {
+
   /**
-   * V-Table for the operation belonging to the tunnel contest.
-   *
-   * Used for all operation specific operations after receiving the ops request
+   * Kept in a DLL of the listener, if @e listener is non-NULL.
    */
-  const struct SetVT *vt;
+  struct Operation *next;
+
+  /**
+   * Kept in a DLL of the listener, if @e listener is non-NULL.
+   */
+  struct Operation *prev;
 
   /**
    * Channel to the peer.
@@ -391,11 +342,15 @@ struct Operation
   struct GNUNET_MQ_Handle *mq;
 
   /**
-   * Detail information about the set operation, including the set to
-   * use.  When 'spec' is NULL, the operation is not yet entirely
-   * initialized.
+   * Context message, may be NULL.
+   */
+  struct GNUNET_MessageHeader *context_msg;
+
+  /**
+   * Set associated with the operation, NULL until the spec has been
+   * associated with a set.
    */
-  struct OperationSpecification *spec;
+  struct Set *set;
 
   /**
    * Operation-specific operation state.  Note that the exact
@@ -405,16 +360,6 @@ struct Operation
   struct OperationState *state;
 
   /**
-   * Evaluate operations are held in a linked list.
-   */
-  struct Operation *next;
-
-  /**
-   * Evaluate operations are held in a linked list.
-   */
-  struct Operation *prev;
-
-  /**
    * The identity of the requesting peer.  Needs to
    * be stored here as the op spec might not have been created yet.
    */
@@ -427,6 +372,50 @@ struct Operation
   struct GNUNET_SCHEDULER_Task *timeout_task;
 
   /**
+   * Salt to use for the operation.
+   */
+  uint32_t salt;
+
+  /**
+   * Remote peers element count
+   */
+  uint32_t remote_element_count;
+
+  /**
+   * ID used to identify an operation between service and client
+   */
+  uint32_t client_request_id;
+
+  /**
+   * When are elements sent to the client, and which elements are sent?
+   */
+  enum GNUNET_SET_ResultMode result_mode;
+
+  /**
+   * Always use delta operation instead of sending full sets,
+   * even it it's less efficient.
+   */
+  int force_delta;
+
+  /**
+   * Always send full sets, even if delta operations would
+   * be more efficient.
+   */
+  int force_full;
+
+  /**
+   * #GNUNET_YES to fail operations where Byzantine faults
+   * are suspected
+   */
+  int byzantine;
+
+  /**
+   * Lower bound for the set size, used only when
+   * byzantine mode is enabled.
+   */
+  int byzantine_lower_bound;
+
+  /**
    * Unique request id for the request from a remote peer, sent to the
    * client, which will accept or reject the request.  Set to '0' iff
    * the request has not been suggested yet.
@@ -434,45 +423,26 @@ struct Operation
   uint32_t suggest_id;
 
   /**
-   * #GNUNET_YES if this is not a "real" set operation yet, and we still
-   * need to wait for the other peer to give us more details.
-   */
-  int is_incoming;
-
-  /**
    * Generation in which the operation handle
    * was created.
    */
   unsigned int generation_created;
 
-  /**
-   * Incremented whenever (during shutdown) some component still
-   * needs to do something with this before the operation is freed.
-   * (Used as a reference counter, but only during termination.)
-   */
-  unsigned int keep;
 };
 
 
 /**
- * SetContent stores the actual set elements,
- * which may be shared by multiple generations derived
- * from one set.
+ * SetContent stores the actual set elements, which may be shared by
+ * multiple generations derived from one set.
  */
 struct SetContent
 {
-  /**
-   * Number of references to the content.
-   */
-  unsigned int refcount;
 
   /**
    * Maps `struct GNUNET_HashCode *` to `struct ElementEntry *`.
    */
   struct GNUNET_CONTAINER_MultiHashMap *elements;
 
-  unsigned int latest_generation;
-
   /**
    * Mutations requested by the client that we're
    * unable to execute right now because we're iterating
@@ -488,6 +458,16 @@ struct SetContent
   struct PendingMutation *pending_mutations_tail;
 
   /**
+   * Number of references to the content.
+   */
+  unsigned int refcount;
+
+  /**
+   * FIXME: document!
+   */
+  unsigned int latest_generation;
+
+  /**
    * Number of concurrently active iterators.
    */
   int iterator_count;
@@ -508,11 +488,24 @@ struct GenerationRange
 };
 
 
+/**
+ * Information about a mutation to apply to a set.
+ */
 struct PendingMutation
 {
+  /**
+   * Mutations are kept in a DLL.
+   */
   struct PendingMutation *prev;
+
+  /**
+   * Mutations are kept in a DLL.
+   */
   struct PendingMutation *next;
 
+  /**
+   * Set this mutation is about.
+   */
   struct Set *set;
 
   /**
@@ -520,7 +513,7 @@ struct PendingMutation
    * May only be a #GNUNET_MESSAGE_TYPE_SET_ADD or
    * #GNUNET_MESSAGE_TYPE_SET_REMOVE.
    */
-  struct GNUNET_MessageHeader *mutation_message;
+  struct GNUNET_SET_ElementMessage *msg;
 };
 
 
@@ -544,12 +537,13 @@ struct Set
    * Client that owns the set.  Only one client may own a set,
    * and there can only be one set per client.
    */
-  struct GNUNET_SERVICE_Client *client;
+  struct ClientState *cs;
 
   /**
-   * Message queue for the client.
+   * Content, possibly shared by multiple sets,
+   * and thus reference counted.
    */
-  struct GNUNET_MQ_Handle *client_mq;
+  struct SetContent *content;
 
   /**
    * Virtual table for this set.  Determined by the operation type of
@@ -582,15 +576,15 @@ struct Set
   struct Operation *ops_tail;
 
   /**
-   * Current generation, that is, number of previously executed
-   * operations and lazy copies on the underlying set content.
+   * List of generations we have to exclude, due to lazy copies.
    */
-  unsigned int current_generation;
+  struct GenerationRange *excluded_generations;
 
   /**
-   * List of generations we have to exclude, due to lazy copies.
+   * Current generation, that is, number of previously executed
+   * operations and lazy copies on the underlying set content.
    */
-  struct GenerationRange *excluded_generations;
+  unsigned int current_generation;
 
   /**
    * Number of elements in array @a excluded_generations.
@@ -603,21 +597,16 @@ struct Set
   enum GNUNET_SET_OperationType operation;
 
   /**
-   * Each @e iter is assigned a unique number, so that the client
-   * can distinguish iterations.
-   */
-  uint16_t iteration_id;
-
-  /**
    * Generation we're currently iteration over.
    */
   unsigned int iter_generation;
 
   /**
-   * Content, possibly shared by multiple sets,
-   * and thus reference counted.
+   * Each @e iter is assigned a unique number, so that the client
+   * can distinguish iterations.
    */
-  struct SetContent *content;
+  uint16_t iteration_id;
+
 };
 
 
@@ -625,10 +614,14 @@ extern struct GNUNET_STATISTICS_Handle *_GSS_statistics;
 
 
 /**
- * Destroy the given operation.  Call the implementation-specific
- * cancel function of the operation.  Disconnects from the remote
- * peer.  Does not disconnect the client, as there may be multiple
- * operations per set.
+ * Destroy the given operation.   Used for any operation where both
+ * peers were known and that thus actually had a vt and channel.  Must
+ * not be used for operations where 'listener' is still set and we do
+ * not know the other peer.
+ *
+ * Call the implementation-specific cancel function of the operation.
+ * Disconnects from the remote peer.  Does not disconnect the client,
+ * as there may be multiple operations per set.
  *
  * @param op operation to destroy
  * @param gc #GNUNET_YES to perform garbage collection on the set
@@ -656,10 +649,13 @@ const struct SetVT *
 _GSS_intersection_vt (void);
 
 
-int
-_GSS_is_element_of_set (struct ElementEntry *ee,
-                        struct Set *set);
-
+/**
+ * Is element @a ee part of the set used by @a op?
+ *
+ * @param ee element to test
+ * @param op operation the defines the set and its generation
+ * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
+ */
 int
 _GSS_is_element_of_operation (struct ElementEntry *ee,
                               struct Operation *op);
diff --git a/src/set/gnunet-service-set_intersection.c b/src/set/gnunet-service-set_intersection.c
index 9fe1eabe6..9dc421792 100644
--- a/src/set/gnunet-service-set_intersection.c
+++ b/src/set/gnunet-service-set_intersection.c
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2013, 2014 GNUnet e.V.
+      Copyright (C) 2013-2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -28,6 +28,7 @@
 #include "gnunet-service-set.h"
 #include "gnunet_block_lib.h"
 #include "gnunet-service-set_protocol.h"
+#include "gnunet-service-set_intersection.h"
 #include <gcrypt.h>
 
 
@@ -55,6 +56,18 @@ enum IntersectionOperationPhase
   PHASE_BF_EXCHANGE,
 
   /**
+   * We must next send the P2P DONE message (after finishing mostly
+   * with the local client).  Then we will wait for the channel to close.
+   */
+  PHASE_MUST_SEND_DONE,
+
+  /**
+   * We have received the P2P DONE message, and must finish with the
+   * local client before terminating the channel.
+   */
+  PHASE_DONE_RECEIVED,
+
+  /**
    * The protocol is over.  Results may still have to be sent to the
    * client.
    */
@@ -161,6 +174,13 @@ struct OperationState
    * Did we send the client that we are done?
    */
   int client_done_sent;
+
+  /**
+   * Set whenever we reach the state where the death of the
+   * channel is perfectly find and should NOT result in the
+   * operation being cancelled.
+   */
+  int channel_death_expected;
 };
 
 
@@ -192,12 +212,12 @@ send_client_removed_element (struct Operation *op,
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_SET_ResultMessage *rm;
 
-  if (GNUNET_SET_RESULT_REMOVED != op->spec->result_mode)
+  if (GNUNET_SET_RESULT_REMOVED != op->result_mode)
     return; /* Wrong mode for transmitting removed elements */
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Sending removed element (size %u) to client\n",
               element->size);
-  GNUNET_assert (0 != op->spec->client_request_id);
+  GNUNET_assert (0 != op->client_request_id);
   ev = GNUNET_MQ_msg_extra (rm,
                             element->size,
                             GNUNET_MESSAGE_TYPE_SET_RESULT);
@@ -207,12 +227,12 @@ send_client_removed_element (struct Operation *op,
     return;
   }
   rm->result_status = htons (GNUNET_SET_STATUS_OK);
-  rm->request_id = htonl (op->spec->client_request_id);
+  rm->request_id = htonl (op->client_request_id);
   rm->element_type = element->element_type;
   GNUNET_memcpy (&rm[1],
-          element->data,
-          element->size);
-  GNUNET_MQ_send (op->spec->set->client_mq,
+                 element->data,
+                 element->size);
+  GNUNET_MQ_send (op->set->cs->mq,
                   ev);
 }
 
@@ -396,9 +416,9 @@ fail_intersection_operation (struct Operation *op)
   ev = GNUNET_MQ_msg (msg,
                       GNUNET_MESSAGE_TYPE_SET_RESULT);
   msg->result_status = htons (GNUNET_SET_STATUS_FAILURE);
-  msg->request_id = htonl (op->spec->client_request_id);
+  msg->request_id = htonl (op->client_request_id);
   msg->element_type = htons (0);
-  GNUNET_MQ_send (op->spec->set->client_mq,
+  GNUNET_MQ_send (op->set->cs->mq,
                   ev);
   _GSS_operation_destroy (op,
                           GNUNET_YES);
@@ -427,8 +447,8 @@ send_bloomfilter (struct Operation *op)
      should use more bits to maximize its set reduction
      potential and minimize overall bandwidth consumption. */
   bf_elementbits = 2 + ceil (log2((double)
-                             (op->spec->remote_element_count /
-                                   (double) op->state->my_element_count)));
+                             (op->remote_element_count /
+                              (double) op->state->my_element_count)));
   if (bf_elementbits < 1)
     bf_elementbits = 1; /* make sure k is not 0 */
   /* optimize BF-size to ~50% of bits set */
@@ -514,12 +534,14 @@ send_client_done_and_destroy (void *cls)
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_SET_ResultMessage *rm;
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Intersection succeeded, sending DONE to local client\n");
   ev = GNUNET_MQ_msg (rm,
                       GNUNET_MESSAGE_TYPE_SET_RESULT);
-  rm->request_id = htonl (op->spec->client_request_id);
+  rm->request_id = htonl (op->client_request_id);
   rm->result_status = htons (GNUNET_SET_STATUS_DONE);
   rm->element_type = htons (0);
-  GNUNET_MQ_send (op->spec->set->client_mq,
+  GNUNET_MQ_send (op->set->cs->mq,
                   ev);
   _GSS_operation_destroy (op,
                           GNUNET_YES);
@@ -527,6 +549,53 @@ send_client_done_and_destroy (void *cls)
 
 
 /**
+ * Remember that we are done dealing with the local client
+ * AND have sent the other peer our message that we are done,
+ * so we are not just waiting for the channel to die before
+ * telling the local client that we are done as our last act.
+ *
+ * @param cls the `struct Operation`.
+ */
+static void
+finished_local_operations (void *cls)
+{
+  struct Operation *op = cls;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "DONE sent to other peer, now waiting for other end to close the channel\n");
+  op->state->phase = PHASE_FINISHED;
+  op->state->channel_death_expected = GNUNET_YES;
+}
+
+
+/**
+ * Notify the other peer that we are done.  Once this message
+ * is out, we still need to notify the local client that we
+ * are done.
+ *
+ * @param op operation to notify for.
+ */
+static void
+send_p2p_done (struct Operation *op)
+{
+  struct GNUNET_MQ_Envelope *ev;
+  struct IntersectionDoneMessage *idm;
+
+  GNUNET_assert (PHASE_MUST_SEND_DONE == op->state->phase);
+  GNUNET_assert (GNUNET_NO == op->state->channel_death_expected);
+  ev = GNUNET_MQ_msg (idm,
+                      GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE);
+  idm->final_element_count = htonl (op->state->my_element_count);
+  idm->element_xor_hash = op->state->my_xor;
+  GNUNET_MQ_notify_sent (ev,
+                         &finished_local_operations,
+                         op);
+  GNUNET_MQ_send (op->mq,
+                  ev);
+}
+
+
+/**
  * Send all elements in the full result iterator.
  *
  * @param cls the `struct Operation *`
@@ -549,8 +618,21 @@ send_remaining_elements (void *cls)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Sending done and destroy because iterator ran out\n");
-    op->keep--;
-    send_client_done_and_destroy (op);
+    GNUNET_CONTAINER_multihashmap_iterator_destroy (op->state->full_result_iter);
+    op->state->full_result_iter = NULL;
+    if (PHASE_DONE_RECEIVED == op->state->phase)
+    {
+      op->state->phase = PHASE_FINISHED;
+      send_client_done_and_destroy (op);
+    }
+    else if (PHASE_MUST_SEND_DONE == op->state->phase)
+    {
+      send_p2p_done (op);
+    }
+    else
+    {
+      GNUNET_assert (0);
+    }
     return;
   }
   ee = nxt;
@@ -559,48 +641,136 @@ send_remaining_elements (void *cls)
               "Sending element %s:%u to client (full set)\n",
               GNUNET_h2s (&ee->element_hash),
               element->size);
-  GNUNET_assert (0 != op->spec->client_request_id);
+  GNUNET_assert (0 != op->client_request_id);
   ev = GNUNET_MQ_msg_extra (rm,
                             element->size,
                             GNUNET_MESSAGE_TYPE_SET_RESULT);
   GNUNET_assert (NULL != ev);
   rm->result_status = htons (GNUNET_SET_STATUS_OK);
-  rm->request_id = htonl (op->spec->client_request_id);
+  rm->request_id = htonl (op->client_request_id);
   rm->element_type = element->element_type;
   GNUNET_memcpy (&rm[1],
-          element->data,
-          element->size);
+                 element->data,
+                 element->size);
   GNUNET_MQ_notify_sent (ev,
                          &send_remaining_elements,
                          op);
-  GNUNET_MQ_send (op->spec->set->client_mq,
+  GNUNET_MQ_send (op->set->cs->mq,
                   ev);
 }
 
 
 /**
- * Inform the peer that this operation is complete.
+ * Fills the "my_elements" hashmap with the initial set of
+ * (non-deleted) elements from the set of the specification.
  *
- * @param op the intersection operation to fail
+ * @param cls closure with the `struct Operation *`
+ * @param key current key code for the element
+ * @param value value in the hash map with the `struct ElementEntry *`
+ * @return #GNUNET_YES (we should continue to iterate)
+ */
+static int
+initialize_map_unfiltered (void *cls,
+                           const struct GNUNET_HashCode *key,
+                           void *value)
+{
+  struct ElementEntry *ee = value;
+  struct Operation *op = cls;
+
+  if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
+    return GNUNET_YES; /* element not live in operation's generation */
+  GNUNET_CRYPTO_hash_xor (&op->state->my_xor,
+                          &ee->element_hash,
+                          &op->state->my_xor);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Initial full initialization of my_elements, adding %s:%u\n",
+              GNUNET_h2s (&ee->element_hash),
+              ee->element.size);
+  GNUNET_break (GNUNET_YES ==
+                GNUNET_CONTAINER_multihashmap_put (op->state->my_elements,
+                                                   &ee->element_hash,
+                                                   ee,
+                                                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  return GNUNET_YES;
+}
+
+
+/**
+ * Send our element count to the peer, in case our element count is
+ * lower than his.
+ *
+ * @param op intersection operation
  */
 static void
-send_peer_done (struct Operation *op)
+send_element_count (struct Operation *op)
 {
   struct GNUNET_MQ_Envelope *ev;
-  struct IntersectionDoneMessage *idm;
+  struct IntersectionElementInfoMessage *msg;
 
-  op->state->phase = PHASE_FINISHED;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Intersection succeeded, sending DONE\n");
-  GNUNET_CONTAINER_bloomfilter_free (op->state->local_bf);
-  op->state->local_bf = NULL;
+              "Sending our element count (%u)\n",
+              op->state->my_element_count);
+  ev = GNUNET_MQ_msg (msg,
+                      GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO);
+  msg->sender_element_count = htonl (op->state->my_element_count);
+  GNUNET_MQ_send (op->mq, ev);
+}
 
-  ev = GNUNET_MQ_msg (idm,
-                      GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE);
-  idm->final_element_count = htonl (op->state->my_element_count);
-  idm->element_xor_hash = op->state->my_xor;
-  GNUNET_MQ_send (op->mq,
-                  ev);
+
+/**
+ * We go first, initialize our map with all elements and
+ * send the first Bloom filter.
+ *
+ * @param op operation to start exchange for
+ */
+static void
+begin_bf_exchange (struct Operation *op)
+{
+  op->state->phase = PHASE_BF_EXCHANGE;
+  GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
+                                         &initialize_map_unfiltered,
+                                         op);
+  send_bloomfilter (op);
+}
+
+
+/**
+ * Handle the initial `struct IntersectionElementInfoMessage` from a
+ * remote peer.
+ *
+ * @param cls the intersection operation
+ * @param mh the header of the message
+ */
+void
+handle_intersection_p2p_element_info (void *cls,
+                                      const struct IntersectionElementInfoMessage *msg)
+{
+  struct Operation *op = cls;
+
+  if (GNUNET_SET_OPERATION_INTERSECTION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    fail_intersection_operation(op);
+    return;
+  }
+  op->remote_element_count = ntohl (msg->sender_element_count);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Received remote element count (%u), I have %u\n",
+              op->remote_element_count,
+              op->state->my_element_count);
+  if ( ( (PHASE_INITIAL != op->state->phase) &&
+         (PHASE_COUNT_SENT != op->state->phase) ) ||
+       (op->state->my_element_count > op->remote_element_count) ||
+       (0 == op->state->my_element_count) ||
+       (0 == op->remote_element_count) )
+  {
+    GNUNET_break_op (0);
+    fail_intersection_operation(op);
+    return;
+  }
+  GNUNET_break (NULL == op->state->remote_bf);
+  begin_bf_exchange (op);
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -615,9 +785,9 @@ process_bf (struct Operation *op)
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received BF in phase %u, foreign count is %u, my element count is %u/%u\n",
               op->state->phase,
-              op->spec->remote_element_count,
+              op->remote_element_count,
               op->state->my_element_count,
-              GNUNET_CONTAINER_multihashmap_size (op->spec->set->content->elements));
+              GNUNET_CONTAINER_multihashmap_size (op->set->content->elements));
   switch (op->state->phase)
   {
   case PHASE_INITIAL:
@@ -627,11 +797,8 @@ process_bf (struct Operation *op)
   case PHASE_COUNT_SENT:
     /* This is the first BF being sent, build our initial map with
        filtering in place */
-    op->state->my_elements
-      = GNUNET_CONTAINER_multihashmap_create (op->spec->remote_element_count,
-                                              GNUNET_YES);
     op->state->my_element_count = 0;
-    GNUNET_CONTAINER_multihashmap_iterate (op->spec->set->content->elements,
+    GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
                                            &filtered_map_initialization,
                                            op);
     break;
@@ -641,6 +808,14 @@ process_bf (struct Operation *op)
                                            &iterator_bf_reduce,
                                            op);
     break;
+  case PHASE_MUST_SEND_DONE:
+    GNUNET_break_op (0);
+    fail_intersection_operation(op);
+    return;
+  case PHASE_DONE_RECEIVED:
+    GNUNET_break_op (0);
+    fail_intersection_operation(op);
+    return;
   case PHASE_FINISHED:
     GNUNET_break_op (0);
     fail_intersection_operation(op);
@@ -650,13 +825,28 @@ process_bf (struct Operation *op)
   op->state->remote_bf = NULL;
 
   if ( (0 == op->state->my_element_count) || /* fully disjoint */
-       ( (op->state->my_element_count == op->spec->remote_element_count) &&
+       ( (op->state->my_element_count == op->remote_element_count) &&
          (0 == memcmp (&op->state->my_xor,
                        &op->state->other_xor,
                        sizeof (struct GNUNET_HashCode))) ) )
   {
     /* we are done */
-    send_peer_done (op);
+    op->state->phase = PHASE_MUST_SEND_DONE;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Intersection succeeded, sending DONE to other peer\n");
+    GNUNET_CONTAINER_bloomfilter_free (op->state->local_bf);
+    op->state->local_bf = NULL;
+    if (GNUNET_SET_RESULT_FULL == op->result_mode)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Sending full result set (%u elements)\n",
+                  GNUNET_CONTAINER_multihashmap_size (op->state->my_elements));
+      op->state->full_result_iter
+        = GNUNET_CONTAINER_multihashmap_iterator_create (op->state->my_elements);
+      send_remaining_elements (op);
+      return;
+    }
+    send_p2p_done (op);
     return;
   }
   op->state->phase = PHASE_BF_EXCHANGE;
@@ -665,41 +855,53 @@ process_bf (struct Operation *op)
 
 
 /**
+ * Check an BF message from a remote peer.
+ *
+ * @param cls the intersection operation
+ * @param msg the header of the message
+ * @return #GNUNET_OK if @a msg is well-formed
+ */
+int
+check_intersection_p2p_bf (void *cls,
+                           const struct BFMessage *msg)
+{
+  struct Operation *op = cls;
+
+  if (GNUNET_SET_OPERATION_INTERSECTION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
  * Handle an BF message from a remote peer.
  *
  * @param cls the intersection operation
- * @param mh the header of the message
+ * @param msg the header of the message
  */
-static void
-handle_p2p_bf (void *cls,
-               const struct GNUNET_MessageHeader *mh)
+void
+handle_intersection_p2p_bf (void *cls,
+                            const struct BFMessage *msg)
 {
   struct Operation *op = cls;
-  const struct BFMessage *msg;
   uint32_t bf_size;
   uint32_t chunk_size;
   uint32_t bf_bits_per_element;
-  uint16_t msize;
 
-  msize = htons (mh->size);
-  if (msize < sizeof (struct BFMessage))
-  {
-    GNUNET_break_op (0);
-    fail_intersection_operation (op);
-    return;
-  }
-  msg = (const struct BFMessage *) mh;
   switch (op->state->phase)
   {
   case PHASE_INITIAL:
     GNUNET_break_op (0);
     fail_intersection_operation (op);
-    break;
+    return;
   case PHASE_COUNT_SENT:
   case PHASE_BF_EXCHANGE:
     bf_size = ntohl (msg->bloomfilter_total_length);
     bf_bits_per_element = ntohl (msg->bits_per_element);
-    chunk_size = msize - sizeof (struct BFMessage);
+    chunk_size = htons (msg->header.size) - sizeof (struct BFMessage);
     op->state->other_xor = msg->element_xor_hash;
     if (bf_size == chunk_size)
     {
@@ -715,9 +917,9 @@ handle_p2p_bf (void *cls,
                                              bf_size,
                                              bf_bits_per_element);
       op->state->salt = ntohl (msg->sender_mutator);
-      op->spec->remote_element_count = ntohl (msg->sender_element_count);
+      op->remote_element_count = ntohl (msg->sender_element_count);
       process_bf (op);
-      return;
+      break;
     }
     /* multipart chunk */
     if (NULL == op->state->bf_data)
@@ -728,7 +930,7 @@ handle_p2p_bf (void *cls,
       op->state->bf_bits_per_element = bf_bits_per_element;
       op->state->bf_data_offset = 0;
       op->state->salt = ntohl (msg->sender_mutator);
-      op->spec->remote_element_count = ntohl (msg->sender_element_count);
+      op->remote_element_count = ntohl (msg->sender_element_count);
     }
     else
     {
@@ -737,7 +939,7 @@ handle_p2p_bf (void *cls,
            (op->state->bf_bits_per_element != bf_bits_per_element) ||
            (op->state->bf_data_offset + chunk_size > bf_size) ||
            (op->state->salt != ntohl (msg->sender_mutator)) ||
-           (op->spec->remote_element_count != ntohl (msg->sender_element_count)) )
+           (op->remote_element_count != ntohl (msg->sender_element_count)) )
       {
         GNUNET_break_op (0);
         fail_intersection_operation (op);
@@ -764,153 +966,9 @@ handle_p2p_bf (void *cls,
   default:
     GNUNET_break_op (0);
     fail_intersection_operation (op);
-    break;
-  }
-}
-
-
-/**
- * Fills the "my_elements" hashmap with the initial set of
- * (non-deleted) elements from the set of the specification.
- *
- * @param cls closure with the `struct Operation *`
- * @param key current key code for the element
- * @param value value in the hash map with the `struct ElementEntry *`
- * @return #GNUNET_YES (we should continue to iterate)
- */
-static int
-initialize_map_unfiltered (void *cls,
-                           const struct GNUNET_HashCode *key,
-                           void *value)
-{
-  struct ElementEntry *ee = value;
-  struct Operation *op = cls;
-
-  if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
-    return GNUNET_YES; /* element not live in operation's generation */
-  GNUNET_CRYPTO_hash_xor (&op->state->my_xor,
-                          &ee->element_hash,
-                          &op->state->my_xor);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Initial full initialization of my_elements, adding %s:%u\n",
-              GNUNET_h2s (&ee->element_hash),
-              ee->element.size);
-  GNUNET_break (GNUNET_YES ==
-                GNUNET_CONTAINER_multihashmap_put (op->state->my_elements,
-                                                   &ee->element_hash,
-                                                   ee,
-                                                   GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  return GNUNET_YES;
-}
-
-
-/**
- * Send our element count to the peer, in case our element count is
- * lower than his.
- *
- * @param op intersection operation
- */
-static void
-send_element_count (struct Operation *op)
-{
-  struct GNUNET_MQ_Envelope *ev;
-  struct IntersectionElementInfoMessage *msg;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Sending our element count (%u)\n",
-              op->state->my_element_count);
-  ev = GNUNET_MQ_msg (msg,
-                      GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO);
-  msg->sender_element_count = htonl (op->state->my_element_count);
-  GNUNET_MQ_send (op->mq, ev);
-}
-
-
-/**
- * We go first, initialize our map with all elements and
- * send the first Bloom filter.
- *
- * @param op operation to start exchange for
- */
-static void
-begin_bf_exchange (struct Operation *op)
-{
-  op->state->phase = PHASE_BF_EXCHANGE;
-  op->state->my_elements
-    = GNUNET_CONTAINER_multihashmap_create (op->state->my_element_count,
-                                            GNUNET_YES);
-  GNUNET_CONTAINER_multihashmap_iterate (op->spec->set->content->elements,
-                                         &initialize_map_unfiltered,
-                                         op);
-  send_bloomfilter (op);
-}
-
-
-/**
- * Handle the initial `struct IntersectionElementInfoMessage` from a
- * remote peer.
- *
- * @param cls the intersection operation
- * @param mh the header of the message
- */
-static void
-handle_p2p_element_info (void *cls,
-                         const struct GNUNET_MessageHeader *mh)
-{
-  struct Operation *op = cls;
-  const struct IntersectionElementInfoMessage *msg;
-
-  if (ntohs (mh->size) != sizeof (struct IntersectionElementInfoMessage))
-  {
-    GNUNET_break_op (0);
-    fail_intersection_operation(op);
-    return;
-  }
-  msg = (const struct IntersectionElementInfoMessage *) mh;
-  op->spec->remote_element_count = ntohl (msg->sender_element_count);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received remote element count (%u), I have %u\n",
-              op->spec->remote_element_count,
-              op->state->my_element_count);
-  if ( ( (PHASE_INITIAL != op->state->phase) &&
-         (PHASE_COUNT_SENT != op->state->phase) ) ||
-       (op->state->my_element_count > op->spec->remote_element_count) ||
-       (0 == op->state->my_element_count) ||
-       (0 == op->spec->remote_element_count) )
-  {
-    GNUNET_break_op (0);
-    fail_intersection_operation(op);
-    return;
-  }
-  GNUNET_break (NULL == op->state->remote_bf);
-  begin_bf_exchange (op);
-}
-
-
-/**
- * Send a result message to the client indicating that the operation
- * is over.  After the result done message has been sent to the
- * client, destroy the evaluate operation.
- *
- * @param op intersection operation
- */
-static void
-finish_and_destroy (struct Operation *op)
-{
-  GNUNET_assert (GNUNET_NO == op->state->client_done_sent);
-
-  if (GNUNET_SET_RESULT_FULL == op->spec->result_mode)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Sending full result set (%u elements)\n",
-                GNUNET_CONTAINER_multihashmap_size (op->state->my_elements));
-    op->state->full_result_iter
-      = GNUNET_CONTAINER_multihashmap_iterator_create (op->state->my_elements);
-    op->keep++;
-    send_remaining_elements (op);
     return;
   }
-  send_client_done_and_destroy (op);
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -955,28 +1013,26 @@ filter_all (void *cls,
  * @param cls the intersection operation
  * @param mh the message
  */
-static void
-handle_p2p_done (void *cls,
-                 const struct GNUNET_MessageHeader *mh)
+void
+handle_intersection_p2p_done (void *cls,
+                              const struct IntersectionDoneMessage *idm)
 {
   struct Operation *op = cls;
-  const struct IntersectionDoneMessage *idm;
 
-  if (PHASE_BF_EXCHANGE != op->state->phase)
+  if (GNUNET_SET_OPERATION_INTERSECTION != op->set->operation)
   {
-    /* wrong phase to conclude? FIXME: Or should we allow this
-       if the other peer has _initially_ already an empty set? */
     GNUNET_break_op (0);
     fail_intersection_operation (op);
     return;
   }
-  if (ntohs (mh->size) != sizeof (struct IntersectionDoneMessage))
+  if (PHASE_BF_EXCHANGE != op->state->phase)
   {
+    /* wrong phase to conclude? FIXME: Or should we allow this
+       if the other peer has _initially_ already an empty set? */
     GNUNET_break_op (0);
     fail_intersection_operation (op);
     return;
   }
-  idm = (const struct IntersectionDoneMessage *) mh;
   if (0 == ntohl (idm->final_element_count))
   {
     /* other peer determined empty set is the intersection,
@@ -998,8 +1054,22 @@ handle_p2p_done (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Got IntersectionDoneMessage, have %u elements in intersection\n",
               op->state->my_element_count);
+  op->state->phase = PHASE_DONE_RECEIVED;
+  GNUNET_CADET_receive_done (op->channel);
+
+  GNUNET_assert (GNUNET_NO == op->state->client_done_sent);
+  if (GNUNET_SET_RESULT_FULL == op->result_mode)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Sending full result set to client (%u elements)\n",
+                GNUNET_CONTAINER_multihashmap_size (op->state->my_elements));
+    op->state->full_result_iter
+      = GNUNET_CONTAINER_multihashmap_iterator_create (op->state->my_elements);
+    send_remaining_elements (op);
+    return;
+  }
   op->state->phase = PHASE_FINISHED;
-  finish_and_destroy (op);
+  send_client_done_and_destroy (op);
 }
 
 
@@ -1010,21 +1080,16 @@ handle_p2p_done (void *cls,
  *        begin the evaluation
  * @param opaque_context message to be transmitted to the listener
  *        to convince him to accept, may be NULL
+ * @return operation-specific state to keep in @a op
  */
-static void
+static struct OperationState *
 intersection_evaluate (struct Operation *op,
                        const struct GNUNET_MessageHeader *opaque_context)
 {
+  struct OperationState *state;
   struct GNUNET_MQ_Envelope *ev;
   struct OperationRequestMessage *msg;
 
-  op->state = GNUNET_new (struct OperationState);
-  /* we started the operation, thus we have to send the operation request */
-  op->state->phase = PHASE_INITIAL;
-  op->state->my_element_count = op->spec->set->state->current_set_element_count;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Initiating intersection operation evaluation\n");
   ev = GNUNET_MQ_msg_nested_mh (msg,
                                 GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
                                 opaque_context);
@@ -1032,20 +1097,30 @@ intersection_evaluate (struct Operation *op,
   {
     /* the context message is too large!? */
     GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (op->spec->set->client);
-    return;
+    return NULL;
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Initiating intersection operation evaluation\n");
+  state = GNUNET_new (struct OperationState);
+  /* we started the operation, thus we have to send the operation request */
+  state->phase = PHASE_INITIAL;
+  state->my_element_count = op->set->state->current_set_element_count;
+  state->my_elements
+    = GNUNET_CONTAINER_multihashmap_create (state->my_element_count,
+                                            GNUNET_YES);
+
   msg->operation = htonl (GNUNET_SET_OPERATION_INTERSECTION);
-  msg->element_count = htonl (op->state->my_element_count);
+  msg->element_count = htonl (state->my_element_count);
   GNUNET_MQ_send (op->mq,
                   ev);
-  op->state->phase = PHASE_COUNT_SENT;
+  state->phase = PHASE_COUNT_SENT;
   if (NULL != opaque_context)
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Sent op request with context message\n");
   else
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Sent op request without context message\n");
+  return state;
 }
 
 
@@ -1055,90 +1130,33 @@ intersection_evaluate (struct Operation *op,
  *
  * @param op operation that will be accepted as an intersection operation
  */
-static void
+static struct OperationState *
 intersection_accept (struct Operation *op)
 {
+  struct OperationState *state;
+
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Accepting set intersection operation\n");
-  op->state = GNUNET_new (struct OperationState);
-  op->state->phase = PHASE_INITIAL;
-  op->state->my_element_count
-    = op->spec->set->state->current_set_element_count;
-  op->state->my_elements
-    = GNUNET_CONTAINER_multihashmap_create
-    (GNUNET_MIN (op->state->my_element_count,
-                 op->spec->remote_element_count),
-     GNUNET_YES);
-  if (op->spec->remote_element_count < op->state->my_element_count)
+  state = GNUNET_new (struct OperationState);
+  state->phase = PHASE_INITIAL;
+  state->my_element_count
+    = op->set->state->current_set_element_count;
+  state->my_elements
+    = GNUNET_CONTAINER_multihashmap_create (GNUNET_MIN (state->my_element_count,
+                                                        op->remote_element_count),
+                                            GNUNET_YES);
+  op->state = state;
+  if (op->remote_element_count < state->my_element_count)
   {
     /* If the other peer (Alice) has fewer elements than us (Bob),
        we just send the count as Alice should send the first BF */
     send_element_count (op);
-    op->state->phase = PHASE_COUNT_SENT;
-    return;
+    state->phase = PHASE_COUNT_SENT;
+    return state;
   }
   /* We have fewer elements, so we start with the BF */
   begin_bf_exchange (op);
-}
-
-
-/**
- * Dispatch messages for a intersection operation.
- *
- * @param op the state of the intersection evaluate operation
- * @param mh the received message
- * @return #GNUNET_SYSERR if the tunnel should be disconnected,
- *         #GNUNET_OK otherwise
- */
-static int
-intersection_handle_p2p_message (struct Operation *op,
-                                 const struct GNUNET_MessageHeader *mh)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Received p2p message (t: %u, s: %u)\n",
-              ntohs (mh->type), ntohs (mh->size));
-  switch (ntohs (mh->type))
-  {
-    /* this message handler is not active until after we received an
-     * operation request message, thus the ops request is not handled here
-     */
-  case GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO:
-    handle_p2p_element_info (op, mh);
-    break;
-  case GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF:
-    handle_p2p_bf (op, mh);
-    break;
-  case GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE:
-    handle_p2p_done (op, mh);
-    break;
-  default:
-    /* something wrong with cadet's message handlers? */
-    GNUNET_assert (0);
-  }
-  return GNUNET_OK;
-}
-
-
-/**
- * Handler for peer-disconnects, notifies the client about the aborted
- * operation.  If we did not expect anything from the other peer, we
- * gracefully terminate the operation.
- *
- * @param op the destroyed operation
- */
-static void
-intersection_peer_disconnect (struct Operation *op)
-{
-  if (PHASE_FINISHED != op->state->phase)
-  {
-    fail_intersection_operation (op);
-    return;
-  }
-  /* the session has already been concluded */
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Other peer disconnected (finished)\n");
-  if (GNUNET_NO == op->state->client_done_sent)
-    finish_and_destroy (op);
+  return state;
 }
 
 
@@ -1168,6 +1186,11 @@ intersection_op_cancel (struct Operation *op)
     GNUNET_CONTAINER_multihashmap_destroy (op->state->my_elements);
     op->state->my_elements = NULL;
   }
+  if (NULL != op->state->full_result_iter)
+  {
+    GNUNET_CONTAINER_multihashmap_iterator_destroy (op->state->full_result_iter);
+    op->state->full_result_iter = NULL;
+  }
   GNUNET_free (op->state);
   op->state = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1236,6 +1259,28 @@ intersection_remove (struct SetState *set_state,
 
 
 /**
+ * Callback for channel death for the intersection operation.
+ *
+ * @param op operation that lost the channel
+ */
+static void
+intersection_channel_death (struct Operation *op)
+{
+  if (GNUNET_YES == op->state->channel_death_expected)
+  {
+    /* oh goodie, we are done! */
+    send_client_done_and_destroy (op);
+  }
+  else
+  {
+    /* sorry, channel went down early, too bad. */
+    _GSS_operation_destroy (op,
+                            GNUNET_YES);
+  }
+}
+
+
+/**
  * Get the table with implementing functions for set intersection.
  *
  * @return the operation specific VTable
@@ -1245,14 +1290,13 @@ _GSS_intersection_vt ()
 {
   static const struct SetVT intersection_vt = {
     .create = &intersection_set_create,
-    .msg_handler = &intersection_handle_p2p_message,
     .add = &intersection_add,
     .remove = &intersection_remove,
     .destroy_set = &intersection_set_destroy,
     .evaluate = &intersection_evaluate,
     .accept = &intersection_accept,
-    .peer_disconnect = &intersection_peer_disconnect,
     .cancel = &intersection_op_cancel,
+    .channel_death = &intersection_channel_death,
   };
 
   return &intersection_vt;
diff --git a/src/set/gnunet-service-set_intersection.h b/src/set/gnunet-service-set_intersection.h
new file mode 100644
index 000000000..3bd255142
--- /dev/null
+++ b/src/set/gnunet-service-set_intersection.h
@@ -0,0 +1,79 @@
+
+/*
+      This file is part of GNUnet
+      Copyright (C) 2013-2017 GNUnet e.V.
+
+      GNUnet is free software; you can redistribute it and/or modify
+      it under the terms of the GNU General Public License as published
+      by the Free Software Foundation; either version 3, or (at your
+      option) any later version.
+
+      GNUnet is distributed in the hope that it will be useful, but
+      WITHOUT ANY WARRANTY; without even the implied warranty of
+      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+      General Public License for more details.
+
+      You should have received a copy of the GNU General Public License
+      along with GNUnet; see the file COPYING.  If not, write to the
+      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+      Boston, MA 02110-1301, USA.
+*/
+/**
+ * @file set/gnunet-service-set_intersection.h
+ * @brief two-peer set operations
+ * @author Florian Dold
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_SET_INTERSECTION_H
+#define GNUNET_SERVICE_SET_INTERSECTION_H
+
+#include "gnunet-service-set.h"
+
+
+/**
+ * Check an BF message from a remote peer.
+ *
+ * @param cls the intersection operation
+ * @param msg the header of the message
+ * @return #GNUNET_OK if @a msg is well-formed
+ */
+int
+check_intersection_p2p_bf (void *cls,
+                           const struct BFMessage *msg);
+
+
+/**
+ * Handle an BF message from a remote peer.
+ *
+ * @param cls the intersection operation
+ * @param msg the header of the message
+ */
+void
+handle_intersection_p2p_bf (void *cls,
+                            const struct BFMessage *msg);
+
+
+/**
+ * Handle the initial `struct IntersectionElementInfoMessage` from a
+ * remote peer.
+ *
+ * @param cls the intersection operation
+ * @param mh the header of the message
+ */
+void
+handle_intersection_p2p_element_info (void *cls,
+                                      const struct IntersectionElementInfoMessage *msg);
+
+
+/**
+ * Handle a done message from a remote peer
+ *
+ * @param cls the intersection operation
+ * @param mh the message
+ */
+void
+handle_intersection_p2p_done (void *cls,
+                              const struct IntersectionDoneMessage *idm);
+
+
+#endif
diff --git a/src/set/gnunet-service-set_union.c b/src/set/gnunet-service-set_union.c
index a19c36206..fc7e578e6 100644
--- a/src/set/gnunet-service-set_union.c
+++ b/src/set/gnunet-service-set_union.c
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      Copyright (C) 2013-2016 GNUnet e.V.
+      Copyright (C) 2013-2017 GNUnet e.V.
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -19,15 +19,16 @@
 */
 /**
  * @file set/gnunet-service-set_union.c
-
  * @brief two-peer set operations
  * @author Florian Dold
+ * @author Christian Grothoff
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_statistics_service.h"
 #include "gnunet-service-set.h"
 #include "ibf.h"
+#include "gnunet-service-set_union.h"
 #include "gnunet-service-set_union_strata_estimator.h"
 #include "gnunet-service-set_protocol.h"
 #include <gcrypt.h>
@@ -367,9 +368,10 @@ fail_union_operation (struct Operation *op)
        "union operation failed\n");
   ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SET_RESULT);
   msg->result_status = htons (GNUNET_SET_STATUS_FAILURE);
-  msg->request_id = htonl (op->spec->client_request_id);
+  msg->request_id = htonl (op->client_request_id);
   msg->element_type = htons (0);
-  GNUNET_MQ_send (op->spec->set->client_mq, ev);
+  GNUNET_MQ_send (op->set->cs->mq,
+                  ev);
   _GSS_operation_destroy (op, GNUNET_YES);
 }
 
@@ -400,7 +402,14 @@ get_ibf_key (const struct GNUNET_HashCode *src)
  */
 struct GetElementContext
 {
+  /**
+   * FIXME.
+   */
   struct GNUNET_HashCode hash;
+
+  /**
+   * FIXME.
+   */
   struct KeyEntry *k;
 };
 
@@ -503,6 +512,9 @@ op_register_element (struct Operation *op,
 }
 
 
+/**
+ * FIXME.
+ */
 static void
 salt_key (const struct IBF_Key *k_in,
           uint32_t salt,
@@ -516,6 +528,9 @@ salt_key (const struct IBF_Key *k_in,
 }
 
 
+/**
+ * FIXME.
+ */
 static void
 unsalt_key (const struct IBF_Key *k_in,
             uint32_t salt,
@@ -549,7 +564,9 @@ prepare_ibf_iterator (void *cls,
        (void *) op,
        (unsigned long) ke->ibf_key.key_val,
        GNUNET_h2s (&ke->element->element_hash));
-  salt_key (&ke->ibf_key, op->state->salt_send, &salted_key);
+  salt_key (&ke->ibf_key,
+            op->state->salt_send,
+            &salted_key);
   ibf_insert (op->state->local_ibf, salted_key);
   return GNUNET_YES;
 }
@@ -575,12 +592,14 @@ init_key_to_element_iterator (void *cls,
 
   /* make sure that the element belongs to the set at the time
    * of creating the operation */
-  if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
+  if (GNUNET_NO ==
+      _GSS_is_element_of_operation (ee,
+                                    op))
     return GNUNET_YES;
-
   GNUNET_assert (GNUNET_NO == ee->remote);
-
-  op_register_element (op, ee, GNUNET_NO);
+  op_register_element (op,
+                       ee,
+                       GNUNET_NO);
   return GNUNET_YES;
 }
 
@@ -597,9 +616,11 @@ initialize_key_to_element (struct Operation *op)
   unsigned int len;
 
   GNUNET_assert (NULL == op->state->key_to_element);
-  len = GNUNET_CONTAINER_multihashmap_size (op->spec->set->content->elements);
+  len = GNUNET_CONTAINER_multihashmap_size (op->set->content->elements);
   op->state->key_to_element = GNUNET_CONTAINER_multihashmap32_create (len + 1);
-  GNUNET_CONTAINER_multihashmap_iterate (op->spec->set->content->elements, init_key_to_element_iterator, op);
+  GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
+                                         &init_key_to_element_iterator,
+                                         op);
 }
 
 
@@ -706,44 +727,6 @@ send_ibf (struct Operation *op,
 
 
 /**
- * Send a strata estimator to the remote peer.
- *
- * @param op the union operation with the remote peer
- */
-static void
-send_strata_estimator (struct Operation *op)
-{
-  const struct StrataEstimator *se = op->state->se;
-  struct GNUNET_MQ_Envelope *ev;
-  struct StrataEstimatorMessage *strata_msg;
-  char *buf;
-  size_t len;
-  uint16_t type;
-
-  buf = GNUNET_malloc (se->strata_count * IBF_BUCKET_SIZE * se->ibf_size);
-  len = strata_estimator_write (op->state->se,
-                                buf);
-  if (len < se->strata_count * IBF_BUCKET_SIZE * se->ibf_size)
-    type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC;
-  else
-    type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE;
-  ev = GNUNET_MQ_msg_extra (strata_msg,
-                            len,
-                            type);
-  GNUNET_memcpy (&strata_msg[1],
-          buf,
-          len);
-  GNUNET_free (buf);
-  strata_msg->set_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap_size (op->spec->set->content->elements));
-  GNUNET_MQ_send (op->mq,
-                  ev);
-  op->state->phase = PHASE_EXPECT_IBF;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "sent SE, expecting IBF\n");
-}
-
-
-/**
  * Compute the necessary order of an ibf
  * from the size of the symmetric set difference.
  *
@@ -761,7 +744,8 @@ get_order_from_difference (unsigned int diff)
     ibf_order++;
   if (ibf_order > MAX_IBF_ORDER)
     ibf_order = MAX_IBF_ORDER;
-  return ibf_order;
+  // add one for correction
+  return ibf_order + 1;
 }
 
 
@@ -775,7 +759,7 @@ get_order_from_difference (unsigned int diff)
  * @return #GNUNET_YES (to continue iterating)
  */
 static int
-send_element_iterator (void *cls,
+send_full_element_iterator (void *cls,
                        const struct GNUNET_HashCode *key,
                        void *value)
 {
@@ -785,26 +769,43 @@ send_element_iterator (void *cls,
   struct GNUNET_SET_Element *el = &ee->element;
   struct GNUNET_MQ_Envelope *ev;
 
-
-  ev = GNUNET_MQ_msg_extra (emsg, el->size, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT);
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Sending element %s\n",
+       GNUNET_h2s (key));
+  ev = GNUNET_MQ_msg_extra (emsg,
+                            el->size,
+                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT);
   emsg->element_type = htons (el->element_type);
-  GNUNET_memcpy (&emsg[1], el->data, el->size);
-  GNUNET_MQ_send (op->mq, ev);
+  GNUNET_memcpy (&emsg[1],
+                 el->data,
+                 el->size);
+  GNUNET_MQ_send (op->mq,
+                  ev);
   return GNUNET_YES;
 }
 
 
+/**
+ * Switch to full set transmission for @a op.
+ *
+ * @param op operation to switch to full set transmission.
+ */
 static void
 send_full_set (struct Operation *op)
 {
   struct GNUNET_MQ_Envelope *ev;
 
   op->state->phase = PHASE_FULL_SENDING;
-
-  (void) GNUNET_CONTAINER_multihashmap_iterate (op->spec->set->content->elements,
-                                                &send_element_iterator, op);
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Dedicing to transmit the full set\n");
+  /* FIXME: use a more memory-friendly way of doing this with an
+     iterator, just as we do in the non-full case! */
+  (void) GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
+                                                &send_full_element_iterator,
+                                                op);
   ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE);
-  GNUNET_MQ_send (op->mq, ev);
+  GNUNET_MQ_send (op->mq,
+                  ev);
 }
 
 
@@ -812,42 +813,56 @@ send_full_set (struct Operation *op)
  * Handle a strata estimator from a remote peer
  *
  * @param cls the union operation
- * @param mh the message
- * @param is_compressed #GNUNET_YES if the estimator is compressed
- * @return #GNUNET_SYSERR if the tunnel should be disconnected,
- *         #GNUNET_OK otherwise
+ * @param msg the message
  */
-static int
-handle_p2p_strata_estimator (void *cls,
-                             const struct GNUNET_MessageHeader *mh,
-                             int is_compressed)
+int
+check_union_p2p_strata_estimator (void *cls,
+                                  const struct StrataEstimatorMessage *msg)
 {
   struct Operation *op = cls;
-  struct StrataEstimator *remote_se;
-  struct StrataEstimatorMessage *msg = (void *) mh;
-  unsigned int diff;
-  uint64_t other_size;
+  int is_compressed;
   size_t len;
 
-  GNUNET_STATISTICS_update (_GSS_statistics,
-                            "# bytes of SE received",
-                            ntohs (mh->size),
-                            GNUNET_NO);
-
   if (op->state->phase != PHASE_EXPECT_SE)
   {
     GNUNET_break (0);
-    fail_union_operation (op);
     return GNUNET_SYSERR;
   }
-  len = ntohs (mh->size) - sizeof (struct StrataEstimatorMessage);
+  is_compressed = (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC == htons (msg->header.type));
+  len = ntohs (msg->header.size) - sizeof (struct StrataEstimatorMessage);
   if ( (GNUNET_NO == is_compressed) &&
        (len != SE_STRATA_COUNT * SE_IBF_SIZE * IBF_BUCKET_SIZE) )
   {
-    fail_union_operation (op);
     GNUNET_break (0);
     return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle a strata estimator from a remote peer
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+void
+handle_union_p2p_strata_estimator (void *cls,
+                                   const struct StrataEstimatorMessage *msg)
+{
+  struct Operation *op = cls;
+  struct StrataEstimator *remote_se;
+  unsigned int diff;
+  uint64_t other_size;
+  size_t len;
+  int is_compressed;
+
+  is_compressed = (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC == htons (msg->header.type));
+  GNUNET_STATISTICS_update (_GSS_statistics,
+                            "# bytes of SE received",
+                            ntohs (msg->header.size),
+                            GNUNET_NO);
+  len = ntohs (msg->header.size) - sizeof (struct StrataEstimatorMessage);
   other_size = GNUNET_ntohll (msg->set_size);
   remote_se = strata_estimator_create (SE_STRATA_COUNT,
                                        SE_IBF_SIZE,
@@ -856,7 +871,7 @@ handle_p2p_strata_estimator (void *cls,
   {
     /* insufficient resources, fail */
     fail_union_operation (op);
-    return GNUNET_SYSERR;
+    return;
   }
   if (GNUNET_OK !=
       strata_estimator_read (&msg[1],
@@ -865,49 +880,81 @@ handle_p2p_strata_estimator (void *cls,
                              remote_se))
   {
     /* decompression failed */
-    fail_union_operation (op);
     strata_estimator_destroy (remote_se);
-    return GNUNET_SYSERR;
+    fail_union_operation (op);
+    return;
   }
   GNUNET_assert (NULL != op->state->se);
   diff = strata_estimator_difference (remote_se,
                                       op->state->se);
+
+  if (diff > 200)
+    diff = diff * 3 / 2;
+
   strata_estimator_destroy (remote_se);
   strata_estimator_destroy (op->state->se);
   op->state->se = NULL;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "got se diff=%d, using ibf size %d\n",
        diff,
-       1<<get_order_from_difference (diff));
+       1U << get_order_from_difference (diff));
+
+  {
+    char *set_debug;
 
-  if ((GNUNET_YES == op->spec->byzantine) && (other_size < op->spec->byzantine_lower_bound))
+    set_debug = getenv ("GNUNET_SET_BENCHMARK");
+    if ( (NULL != set_debug) &&
+         (0 == strcmp (set_debug, "1")) )
+    {
+      FILE *f = fopen ("set.log", "a");
+      fprintf (f, "%llu\n", (unsigned long long) diff);
+      fclose (f);
+    }
+  }
+
+  if ( (GNUNET_YES == op->byzantine) &&
+       (other_size < op->byzantine_lower_bound) )
   {
     GNUNET_break (0);
     fail_union_operation (op);
-    return GNUNET_SYSERR;
+    return;
   }
 
-
-  if ( (GNUNET_YES == op->spec->force_full) || (diff > op->state->initial_size / 2))
+  if ( (GNUNET_YES == op->force_full) ||
+       (diff > op->state->initial_size / 4) ||
+       (0 == other_size) )
   {
     LOG (GNUNET_ERROR_TYPE_INFO,
-         "Sending full set (diff=%d, own set=%u)\n",
+         "Deciding to go for full set transmission (diff=%d, own set=%u)\n",
          diff,
          op->state->initial_size);
-    if (op->state->initial_size <= other_size)
+    GNUNET_STATISTICS_update (_GSS_statistics,
+                              "# of full sends",
+                              1,
+                              GNUNET_NO);
+    if ( (op->state->initial_size <= other_size) ||
+         (0 == other_size) )
     {
       send_full_set (op);
     }
     else
     {
       struct GNUNET_MQ_Envelope *ev;
+
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           "Telling other peer that we expect its full set\n");
       op->state->phase = PHASE_EXPECT_IBF;
       ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL);
-      GNUNET_MQ_send (op->mq, ev);
+      GNUNET_MQ_send (op->mq,
+                      ev);
     }
   }
   else
   {
+    GNUNET_STATISTICS_update (_GSS_statistics,
+                              "# of ibf sends",
+                              1,
+                              GNUNET_NO);
     if (GNUNET_OK !=
         send_ibf (op,
                   get_order_from_difference (diff)))
@@ -916,11 +963,10 @@ handle_p2p_strata_estimator (void *cls,
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   "Failed to send IBF, closing connection\n");
       fail_union_operation (op);
-      return GNUNET_SYSERR;
+      return;
     }
   }
-
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -1001,14 +1047,16 @@ decode_and_send (struct Operation *op)
   GNUNET_assert (PHASE_INVENTORY_ACTIVE == op->state->phase);
 
   if (GNUNET_OK !=
-      prepare_ibf (op, op->state->remote_ibf->size))
+      prepare_ibf (op,
+                   op->state->remote_ibf->size))
   {
     GNUNET_break (0);
     /* allocation failed */
     return GNUNET_SYSERR;
   }
   diff_ibf = ibf_dup (op->state->local_ibf);
-  ibf_subtract (diff_ibf, op->state->remote_ibf);
+  ibf_subtract (diff_ibf,
+                op->state->remote_ibf);
 
   ibf_destroy (op->state->remote_ibf);
   op->state->remote_ibf = NULL;
@@ -1105,8 +1153,12 @@ decode_and_send (struct Operation *op)
     if (1 == side)
     {
       struct IBF_Key unsalted_key;
-      unsalt_key (&key, op->state->salt_receive, &unsalted_key);
-      send_offers_for_key (op, unsalted_key);
+
+      unsalt_key (&key,
+                  op->state->salt_receive,
+                  &unsalted_key);
+      send_offers_for_key (op,
+                           unsalted_key);
     }
     else if (-1 == side)
     {
@@ -1138,99 +1190,118 @@ decode_and_send (struct Operation *op)
 
 
 /**
- * Handle an IBF message from a remote peer.
+ * Check an IBF message from a remote peer.
  *
  * Reassemble the IBF from multiple pieces, and
  * process the whole IBF once possible.
  *
  * @param cls the union operation
- * @param mh the header of the message
- * @return #GNUNET_SYSERR if the tunnel should be disconnected,
- *         #GNUNET_OK otherwise
+ * @param msg the header of the message
+ * @return #GNUNET_OK if @a msg is well-formed
  */
-static int
-handle_p2p_ibf (void *cls,
-                const struct GNUNET_MessageHeader *mh)
+int
+check_union_p2p_ibf (void *cls,
+                     const struct IBFMessage *msg)
 {
   struct Operation *op = cls;
-  const struct IBFMessage *msg;
   unsigned int buckets_in_message;
 
-  if (ntohs (mh->size) < sizeof (struct IBFMessage))
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
     return GNUNET_SYSERR;
   }
-  msg = (const struct IBFMessage *) mh;
-  if ( (op->state->phase == PHASE_INVENTORY_PASSIVE) ||
-       (op->state->phase == PHASE_EXPECT_IBF) )
+  buckets_in_message = (ntohs (msg->header.size) - sizeof *msg) / IBF_BUCKET_SIZE;
+  if (0 == buckets_in_message)
   {
-    op->state->phase = PHASE_EXPECT_IBF_CONT;
-    GNUNET_assert (NULL == op->state->remote_ibf);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Creating new ibf of size %u\n",
-         1 << msg->order);
-    op->state->remote_ibf = ibf_create (1<<msg->order, SE_IBF_HASH_NUM);
-    op->state->salt_receive = ntohl (msg->salt);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Receiving new IBF with salt %u\n", op->state->salt_receive);
-    if (NULL == op->state->remote_ibf)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  "Failed to parse remote IBF, closing connection\n");
-      fail_union_operation (op);
-      return GNUNET_SYSERR;
-    }
-    op->state->ibf_buckets_received = 0;
-    if (0 != ntohl (msg->offset))
-    {
-      GNUNET_break_op (0);
-      fail_union_operation (op);
-      return GNUNET_SYSERR;
-    }
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
+  if ((ntohs (msg->header.size) - sizeof *msg) != buckets_in_message * IBF_BUCKET_SIZE)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
-  else if (op->state->phase == PHASE_EXPECT_IBF_CONT)
+  if (op->state->phase == PHASE_EXPECT_IBF_CONT)
   {
     if (ntohl (msg->offset) != op->state->ibf_buckets_received)
     {
       GNUNET_break_op (0);
-      fail_union_operation (op);
       return GNUNET_SYSERR;
     }
     if (1<<msg->order != op->state->remote_ibf->size)
     {
       GNUNET_break_op (0);
-      fail_union_operation (op);
       return GNUNET_SYSERR;
     }
     if (ntohl (msg->salt) != op->state->salt_receive)
     {
       GNUNET_break_op (0);
-      fail_union_operation (op);
       return GNUNET_SYSERR;
     }
   }
-  else
+  else if ( (op->state->phase != PHASE_INVENTORY_PASSIVE) &&
+            (op->state->phase != PHASE_EXPECT_IBF) )
   {
-    GNUNET_assert (0);
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
 
-  buckets_in_message = (ntohs (msg->header.size) - sizeof *msg) / IBF_BUCKET_SIZE;
+  return GNUNET_OK;
+}
 
-  if (0 == buckets_in_message)
+
+/**
+ * Handle an IBF message from a remote peer.
+ *
+ * Reassemble the IBF from multiple pieces, and
+ * process the whole IBF once possible.
+ *
+ * @param cls the union operation
+ * @param msg the header of the message
+ */
+void
+handle_union_p2p_ibf (void *cls,
+                      const struct IBFMessage *msg)
+{
+  struct Operation *op = cls;
+  unsigned int buckets_in_message;
+
+  buckets_in_message = (ntohs (msg->header.size) - sizeof *msg) / IBF_BUCKET_SIZE;
+  if ( (op->state->phase == PHASE_INVENTORY_PASSIVE) ||
+       (op->state->phase == PHASE_EXPECT_IBF) )
   {
-    GNUNET_break_op (0);
-    fail_union_operation (op);
-    return GNUNET_SYSERR;
+    op->state->phase = PHASE_EXPECT_IBF_CONT;
+    GNUNET_assert (NULL == op->state->remote_ibf);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Creating new ibf of size %u\n",
+         1 << msg->order);
+    op->state->remote_ibf = ibf_create (1<<msg->order, SE_IBF_HASH_NUM);
+    op->state->salt_receive = ntohl (msg->salt);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Receiving new IBF with salt %u\n",
+         op->state->salt_receive);
+    if (NULL == op->state->remote_ibf)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  "Failed to parse remote IBF, closing connection\n");
+      fail_union_operation (op);
+      return;
+    }
+    op->state->ibf_buckets_received = 0;
+    if (0 != ntohl (msg->offset))
+    {
+      GNUNET_break_op (0);
+      fail_union_operation (op);
+      return;
+    }
   }
-
-  if ((ntohs (msg->header.size) - sizeof *msg) != buckets_in_message * IBF_BUCKET_SIZE)
+  else
   {
-    GNUNET_break_op (0);
-    fail_union_operation (op);
-    return GNUNET_SYSERR;
+    GNUNET_assert (op->state->phase == PHASE_EXPECT_IBF_CONT);
+    LOG (GNUNET_ERROR_TYPE_INFO,
+         "Received more of IBF\n");
   }
-
   GNUNET_assert (NULL != op->state->remote_ibf);
 
   ibf_read_slice (&msg[1],
@@ -1250,10 +1321,11 @@ handle_p2p_ibf (void *cls,
       /* Internal error, best we can do is shut down */
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   "Failed to decode IBF, closing connection\n");
-      return GNUNET_SYSERR;
+      fail_union_operation (op);
+      return;
     }
   }
-  return GNUNET_OK;
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -1276,7 +1348,7 @@ send_client_element (struct Operation *op,
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "sending element (size %u) to client\n",
        element->size);
-  GNUNET_assert (0 != op->spec->client_request_id);
+  GNUNET_assert (0 != op->client_request_id);
   ev = GNUNET_MQ_msg_extra (rm, element->size, GNUNET_MESSAGE_TYPE_SET_RESULT);
   if (NULL == ev)
   {
@@ -1285,11 +1357,14 @@ send_client_element (struct Operation *op,
     return;
   }
   rm->result_status = htons (status);
-  rm->request_id = htonl (op->spec->client_request_id);
+  rm->request_id = htonl (op->client_request_id);
   rm->element_type = htons (element->element_type);
   rm->current_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap32_size (op->state->key_to_element));
-  GNUNET_memcpy (&rm[1], element->data, element->size);
-  GNUNET_MQ_send (op->spec->set->client_mq, ev);
+  GNUNET_memcpy (&rm[1],
+                 element->data,
+                 element->size);
+  GNUNET_MQ_send (op->set->cs->mq,
+                  ev);
 }
 
 
@@ -1306,17 +1381,27 @@ send_done_and_destroy (void *cls)
   struct GNUNET_MQ_Envelope *ev;
   struct GNUNET_SET_ResultMessage *rm;
 
-  ev = GNUNET_MQ_msg (rm, GNUNET_MESSAGE_TYPE_SET_RESULT);
-  rm->request_id = htonl (op->spec->client_request_id);
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Signalling client that union operation is done\n");
+  ev = GNUNET_MQ_msg (rm,
+                      GNUNET_MESSAGE_TYPE_SET_RESULT);
+  rm->request_id = htonl (op->client_request_id);
   rm->result_status = htons (GNUNET_SET_STATUS_DONE);
   rm->element_type = htons (0);
   rm->current_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap32_size (op->state->key_to_element));
-  GNUNET_MQ_send (op->spec->set->client_mq, ev);
+  GNUNET_MQ_send (op->set->cs->mq,
+                  ev);
   /* Will also call the union-specific cancel function. */
-  _GSS_operation_destroy (op, GNUNET_YES);
+  _GSS_operation_destroy (op,
+                          GNUNET_YES);
 }
 
 
+/**
+ * Tests if the operation is finished, and if so notify.
+ *
+ * @param op operation to check
+ */
 static void
 maybe_finish (struct Operation *op)
 {
@@ -1335,8 +1420,8 @@ maybe_finish (struct Operation *op)
 
       op->state->phase = PHASE_DONE;
       ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE);
-      GNUNET_MQ_send (op->mq, ev);
-
+      GNUNET_MQ_send (op->mq,
+                      ev);
       /* We now wait until the other peer closes the channel
        * after it got all elements from us. */
     }
@@ -1356,46 +1441,59 @@ maybe_finish (struct Operation *op)
 
 
 /**
- * Handle an element message from a remote peer.
- * Sent by the other peer either because we decoded an IBF and placed a demand,
- * or because the other peer switched to full set transmission.
+ * Check an element message from a remote peer.
  *
  * @param cls the union operation
- * @param mh the message
+ * @param emsg the message
  */
-static void
-handle_p2p_elements (void *cls,
-                     const struct GNUNET_MessageHeader *mh)
+int
+check_union_p2p_elements (void *cls,
+                          const struct GNUNET_SET_ElementMessage *emsg)
 {
   struct Operation *op = cls;
-  struct ElementEntry *ee;
-  const struct GNUNET_SET_ElementMessage *emsg;
-  uint16_t element_size;
 
-  if (0 == GNUNET_CONTAINER_multihashmap_size (op->state->demanded_hashes))
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
-  if (ntohs (mh->size) < sizeof (struct GNUNET_SET_ElementMessage))
+  if (0 == GNUNET_CONTAINER_multihashmap_size (op->state->demanded_hashes))
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
 
-  emsg = (const struct GNUNET_SET_ElementMessage *) mh;
 
-  element_size = ntohs (mh->size) - sizeof (struct GNUNET_SET_ElementMessage);
+/**
+ * Handle an element message from a remote peer.
+ * Sent by the other peer either because we decoded an IBF and placed a demand,
+ * or because the other peer switched to full set transmission.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+void
+handle_union_p2p_elements (void *cls,
+                           const struct GNUNET_SET_ElementMessage *emsg)
+{
+  struct Operation *op = cls;
+  struct ElementEntry *ee;
+  struct KeyEntry *ke;
+  uint16_t element_size;
+
+  element_size = ntohs (emsg->header.size) - sizeof (struct GNUNET_SET_ElementMessage);
   ee = GNUNET_malloc (sizeof (struct ElementEntry) + element_size);
-  GNUNET_memcpy (&ee[1], &emsg[1], element_size);
+  GNUNET_memcpy (&ee[1],
+                 &emsg[1],
+                 element_size);
   ee->element.size = element_size;
   ee->element.data = &ee[1];
   ee->element.element_type = ntohs (emsg->element_type);
   ee->remote = GNUNET_YES;
-  GNUNET_SET_element_hash (&ee->element, &ee->element_hash);
-
+  GNUNET_SET_element_hash (&ee->element,
+                           &ee->element_hash);
   if (GNUNET_NO ==
       GNUNET_CONTAINER_multihashmap_remove (op->state->demanded_hashes,
                                             &ee->element_hash,
@@ -1403,7 +1501,6 @@ handle_p2p_elements (void *cls,
   {
     /* We got something we didn't demand, since it's not in our map. */
     GNUNET_break_op (0);
-    GNUNET_free (ee);
     fail_union_operation (op);
     return;
   }
@@ -1422,10 +1519,9 @@ handle_p2p_elements (void *cls,
                             1,
                             GNUNET_NO);
 
-  op->state->received_total += 1;
-
-  struct KeyEntry *ke = op_get_element (op, &ee->element_hash);
+  op->state->received_total++;
 
+  ke = op_get_element (op, &ee->element_hash);
   if (NULL != ke)
   {
     /* Got repeated element.  Should not happen since
@@ -1441,10 +1537,10 @@ handle_p2p_elements (void *cls,
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Registering new element from remote peer\n");
-    op->state->received_fresh += 1;
+    op->state->received_fresh++;
     op_register_element (op, ee, GNUNET_YES);
     /* only send results immediately if the client wants it */
-    switch (op->spec->result_mode)
+    switch (op->result_mode)
     {
       case GNUNET_SET_RESULT_ADDED:
         send_client_element (op, &ee->element, GNUNET_SET_STATUS_OK);
@@ -1459,43 +1555,57 @@ handle_p2p_elements (void *cls,
     }
   }
 
-  if (op->state->received_total > 8 && op->state->received_fresh < op->state->received_total / 3)
+  if ( (op->state->received_total > 8) &&
+       (op->state->received_fresh < op->state->received_total / 3) )
   {
     /* The other peer gave us lots of old elements, there's something wrong. */
     GNUNET_break_op (0);
     fail_union_operation (op);
     return;
   }
-
+  GNUNET_CADET_receive_done (op->channel);
   maybe_finish (op);
 }
 
 
 /**
- * Handle an element message from a remote peer.
+ * Check a full element message from a remote peer.
  *
  * @param cls the union operation
- * @param mh the message
+ * @param emsg the message
  */
-static void
-handle_p2p_full_element (void *cls,
-                         const struct GNUNET_MessageHeader *mh)
+int
+check_union_p2p_full_element (void *cls,
+                              const struct GNUNET_SET_ElementMessage *emsg)
 {
   struct Operation *op = cls;
-  struct ElementEntry *ee;
-  const struct GNUNET_SET_ElementMessage *emsg;
-  uint16_t element_size;
 
-  if (ntohs (mh->size) < sizeof (struct GNUNET_SET_ElementMessage))
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
+  // FIXME: check that we expect full elements here?
+  return GNUNET_OK;
+}
 
-  emsg = (const struct GNUNET_SET_ElementMessage *) mh;
 
-  element_size = ntohs (mh->size) - sizeof (struct GNUNET_SET_ElementMessage);
+/**
+ * Handle an element message from a remote peer.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+void
+handle_union_p2p_full_element (void *cls,
+                               const struct GNUNET_SET_ElementMessage *emsg)
+{
+  struct Operation *op = cls;
+  struct ElementEntry *ee;
+  struct KeyEntry *ke;
+  uint16_t element_size;
+
+  element_size = ntohs (emsg->header.size) - sizeof (struct GNUNET_SET_ElementMessage);
   ee = GNUNET_malloc (sizeof (struct ElementEntry) + element_size);
   GNUNET_memcpy (&ee[1], &emsg[1], element_size);
   ee->element.size = element_size;
@@ -1518,10 +1628,9 @@ handle_p2p_full_element (void *cls,
                             1,
                             GNUNET_NO);
 
-  op->state->received_total += 1;
-
-  struct KeyEntry *ke = op_get_element (op, &ee->element_hash);
+  op->state->received_total++;
 
+  ke = op_get_element (op, &ee->element_hash);
   if (NULL != ke)
   {
     /* Got repeated element.  Should not happen since
@@ -1537,10 +1646,10 @@ handle_p2p_full_element (void *cls,
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Registering new element from remote peer\n");
-    op->state->received_fresh += 1;
+    op->state->received_fresh++;
     op_register_element (op, ee, GNUNET_YES);
     /* only send results immediately if the client wants it */
-    switch (op->spec->result_mode)
+    switch (op->result_mode)
     {
       case GNUNET_SET_RESULT_ADDED:
         send_client_element (op, &ee->element, GNUNET_SET_STATUS_OK);
@@ -1555,9 +1664,9 @@ handle_p2p_full_element (void *cls,
     }
   }
 
-  if ( (GNUNET_YES == op->spec->byzantine) && 
-       (op->state->received_total > 150) && 
-       (op->state->received_fresh < op->state->received_total / 3) )
+  if ( (GNUNET_YES == op->byzantine) &&
+       (op->state->received_total > 384 + op->state->received_fresh * 4) &&
+       (op->state->received_fresh < op->state->received_total / 6) )
   {
     /* The other peer gave us lots of old elements, there's something wrong. */
     LOG (GNUNET_ERROR_TYPE_ERROR,
@@ -1568,51 +1677,78 @@ handle_p2p_full_element (void *cls,
     fail_union_operation (op);
     return;
   }
+  GNUNET_CADET_receive_done (op->channel);
 }
 
+
 /**
  * Send offers (for GNUNET_Hash-es) in response
  * to inquiries (for IBF_Key-s).
  *
  * @param cls the union operation
- * @param mh the message
+ * @param msg the message
  */
-static void
-handle_p2p_inquiry (void *cls,
-                    const struct GNUNET_MessageHeader *mh)
+int
+check_union_p2p_inquiry (void *cls,
+                         const struct InquiryMessage *msg)
 {
   struct Operation *op = cls;
-  const struct IBF_Key *ibf_key;
   unsigned int num_keys;
-  struct InquiryMessage *msg;
 
-  /* look up elements and send them */
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
   if (op->state->phase != PHASE_INVENTORY_PASSIVE)
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
-  num_keys = (ntohs (mh->size) - sizeof (struct InquiryMessage))
-      / sizeof (struct IBF_Key);
-  if ((ntohs (mh->size) - sizeof (struct InquiryMessage))
+  num_keys = (ntohs (msg->header.size) - sizeof (struct InquiryMessage))
+    / sizeof (struct IBF_Key);
+  if ((ntohs (msg->header.size) - sizeof (struct InquiryMessage))
       != num_keys * sizeof (struct IBF_Key))
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
 
-  msg = (struct InquiryMessage *) mh;
 
+/**
+ * Send offers (for GNUNET_Hash-es) in response
+ * to inquiries (for IBF_Key-s).
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+void
+handle_union_p2p_inquiry (void *cls,
+                          const struct InquiryMessage *msg)
+{
+  struct Operation *op = cls;
+  const struct IBF_Key *ibf_key;
+  unsigned int num_keys;
+
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Received union inquiry\n");
+  num_keys = (ntohs (msg->header.size) - sizeof (struct InquiryMessage))
+    / sizeof (struct IBF_Key);
   ibf_key = (const struct IBF_Key *) &msg[1];
   while (0 != num_keys--)
   {
     struct IBF_Key unsalted_key;
-    unsalt_key (ibf_key, ntohl (msg->salt), &unsalted_key);
-    send_offers_for_key (op, unsalted_key);
+
+    unsalt_key (ibf_key,
+                ntohl (msg->salt),
+                &unsalted_key);
+    send_offers_for_key (op,
+                         unsalted_key);
     ibf_key++;
   }
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -1627,9 +1763,9 @@ handle_p2p_inquiry (void *cls,
  *         #GNUNET_NO if not.
  */
 static int
-send_missing_elements_iter (void *cls,
-                            uint32_t key,
-                            void *value)
+send_missing_full_elements_iter (void *cls,
+                                 uint32_t key,
+                                 void *value)
 {
   struct Operation *op = cls;
   struct KeyEntry *ke = value;
@@ -1639,39 +1775,50 @@ send_missing_elements_iter (void *cls,
 
   if (GNUNET_YES == ke->received)
     return GNUNET_YES;
-
-  ev = GNUNET_MQ_msg_extra (emsg, ee->element.size, GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT);
-  GNUNET_memcpy (&emsg[1], ee->element.data, ee->element.size);
-  emsg->reserved = htons (0);
+  ev = GNUNET_MQ_msg_extra (emsg,
+                            ee->element.size,
+                            GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT);
+  GNUNET_memcpy (&emsg[1],
+                 ee->element.data,
+                 ee->element.size);
   emsg->element_type = htons (ee->element.element_type);
-  GNUNET_MQ_send (op->mq, ev);
-
+  GNUNET_MQ_send (op->mq,
+                  ev);
   return GNUNET_YES;
 }
 
 
 /**
- * Handle a 
+ * Handle a request for full set transmission.
  *
  * @parem cls closure, a set union operation
  * @param mh the demand message
  */
-static void
-handle_p2p_request_full (void *cls,
-                         const struct GNUNET_MessageHeader *mh)
+void
+handle_union_p2p_request_full (void *cls,
+                               const struct GNUNET_MessageHeader *mh)
 {
   struct Operation *op = cls;
 
-  if (PHASE_EXPECT_IBF != op->state->phase)
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Received request for full set transmission\n");
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
   {
+    GNUNET_break_op (0);
     fail_union_operation (op);
+    return;
+  }
+  if (PHASE_EXPECT_IBF != op->state->phase)
+  {
     GNUNET_break_op (0);
+    fail_union_operation (op);
     return;
   }
 
   // FIXME: we need to check that our set is larger than the
   // byzantine_lower_bound by some threshold
   send_full_set (op);
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -1681,56 +1828,101 @@ handle_p2p_request_full (void *cls,
  * @parem cls closure, a set union operation
  * @param mh the demand message
  */
-static void
-handle_p2p_full_done (void *cls,
-                      const struct GNUNET_MessageHeader *mh)
+void
+handle_union_p2p_full_done (void *cls,
+                            const struct GNUNET_MessageHeader *mh)
 {
   struct Operation *op = cls;
 
-  if (PHASE_EXPECT_IBF == op->state->phase)
+  switch (op->state->phase)
   {
-    struct GNUNET_MQ_Envelope *ev;
+  case PHASE_EXPECT_IBF:
+    {
+      struct GNUNET_MQ_Envelope *ev;
 
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "got FULL DONE, sending elements that other peer is missing\n");
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "got FULL DONE, sending elements that other peer is missing\n");
+
+      /* send all the elements that did not come from the remote peer */
+      GNUNET_CONTAINER_multihashmap32_iterate (op->state->key_to_element,
+                                               &send_missing_full_elements_iter,
+                                               op);
+
+      ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE);
+      GNUNET_MQ_notify_sent (ev,
+                             &send_done_and_destroy,
+                             op);
+      GNUNET_MQ_send (op->mq,
+                      ev);
+      op->state->phase = PHASE_DONE;
+      /* we now wait until the other peer shuts the tunnel down*/
+    }
+    break;
+  case PHASE_FULL_SENDING:
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "got FULL DONE, finishing\n");
+      /* We sent the full set, and got the response for that.  We're done. */
+      op->state->phase = PHASE_DONE;
+      GNUNET_CADET_receive_done (op->channel);
+      send_done_and_destroy (op);
+      return;
+    }
+    break;
+  default:
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Handle full done phase is %u\n",
+                (unsigned) op->state->phase);
+    GNUNET_break_op (0);
+    fail_union_operation (op);
+    return;
+  }
+  GNUNET_CADET_receive_done (op->channel);
+}
 
-    /* send all the elements that did not come from the remote peer */
-    GNUNET_CONTAINER_multihashmap32_iterate (op->state->key_to_element,
-                                             &send_missing_elements_iter,
-                                             op);
 
-    ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE);
-    GNUNET_MQ_send (op->mq, ev);
-    op->state->phase = PHASE_DONE;
+/**
+ * Check a demand by the other peer for elements based on a list
+ * of `struct GNUNET_HashCode`s.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ * @return #GNUNET_OK if @a mh is well-formed
+ */
+int
+check_union_p2p_demand (void *cls,
+                        const struct GNUNET_MessageHeader *mh)
+{
+  struct Operation *op = cls;
+  unsigned int num_hashes;
 
-    /* we now wait until the other peer shuts the tunnel down*/
-  }
-  else if (PHASE_FULL_SENDING == op->state->phase)
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "got FULL DONE, finishing\n");
-    /* We sent the full set, and got the response for that.  We're done. */
-    op->state->phase = PHASE_DONE;
-    send_done_and_destroy (op);
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
   }
-  else
+  num_hashes = (ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
+    / sizeof (struct GNUNET_HashCode);
+  if ((ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
+      != num_hashes * sizeof (struct GNUNET_HashCode))
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO, "handle full done phase is %u\n", (unsigned) op->state->phase);
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
 }
 
 
 /**
  * Handle a demand by the other peer for elements based on a list
- * of GNUNET_HashCode-s.
+ * of `struct GNUNET_HashCode`s.
  *
  * @parem cls closure, a set union operation
  * @param mh the demand message
  */
-static void
-handle_p2p_demand (void *cls,
-                   const struct GNUNET_MessageHeader *mh)
+void
+handle_union_p2p_demand (void *cls,
+                         const struct GNUNET_MessageHeader *mh)
 {
   struct Operation *op = cls;
   struct ElementEntry *ee;
@@ -1741,19 +1933,12 @@ handle_p2p_demand (void *cls,
 
   num_hashes = (ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
     / sizeof (struct GNUNET_HashCode);
-  if ((ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
-      != num_hashes * sizeof (struct GNUNET_HashCode))
-  {
-    GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
-  }
-
   for (hash = (const struct GNUNET_HashCode *) &mh[1];
        num_hashes > 0;
        hash++, num_hashes--)
   {
-    ee = GNUNET_CONTAINER_multihashmap_get (op->spec->set->content->elements, hash);
+    ee = GNUNET_CONTAINER_multihashmap_get (op->set->content->elements,
+                                            hash);
     if (NULL == ee)
     {
       /* Demand for non-existing element. */
@@ -1783,7 +1968,7 @@ handle_p2p_demand (void *cls,
                               1,
                               GNUNET_NO);
 
-    switch (op->spec->result_mode)
+    switch (op->result_mode)
     {
       case GNUNET_SET_RESULT_ADDED:
         /* Nothing to do. */
@@ -1797,42 +1982,65 @@ handle_p2p_demand (void *cls,
         break;
     }
   }
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
 /**
- * Handle offers (of GNUNET_HashCode-s) and
- * respond with demands (of GNUNET_HashCode-s).
+ * Check offer (of `struct GNUNET_HashCode`s).
  *
  * @param cls the union operation
  * @param mh the message
+ * @return #GNUNET_OK if @a mh is well-formed
  */
-static void
-handle_p2p_offer (void *cls,
-                    const struct GNUNET_MessageHeader *mh)
+int
+check_union_p2p_offer (void *cls,
+                        const struct GNUNET_MessageHeader *mh)
 {
   struct Operation *op = cls;
-  const struct GNUNET_HashCode *hash;
   unsigned int num_hashes;
 
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    return GNUNET_SYSERR;
+  }
   /* look up elements and send them */
   if ( (op->state->phase != PHASE_INVENTORY_PASSIVE) &&
        (op->state->phase != PHASE_INVENTORY_ACTIVE))
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
   num_hashes = (ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
     / sizeof (struct GNUNET_HashCode);
-  if ((ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
-      != num_hashes * sizeof (struct GNUNET_HashCode))
+  if ((ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader)) !=
+      num_hashes * sizeof (struct GNUNET_HashCode))
   {
     GNUNET_break_op (0);
-    fail_union_operation (op);
-    return;
+    return GNUNET_SYSERR;
   }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Handle offers (of `struct GNUNET_HashCode`s) and
+ * respond with demands (of `struct GNUNET_HashCode`s).
+ *
+ * @param cls the union operation
+ * @param mh the message
+ */
+void
+handle_union_p2p_offer (void *cls,
+                        const struct GNUNET_MessageHeader *mh)
+{
+  struct Operation *op = cls;
+  const struct GNUNET_HashCode *hash;
+  unsigned int num_hashes;
 
+  num_hashes = (ntohs (mh->size) - sizeof (struct GNUNET_MessageHeader))
+    / sizeof (struct GNUNET_HashCode);
   for (hash = (const struct GNUNET_HashCode *) &mh[1];
        num_hashes > 0;
        hash++, num_hashes--)
@@ -1841,7 +2049,7 @@ handle_p2p_offer (void *cls,
     struct GNUNET_MessageHeader *demands;
     struct GNUNET_MQ_Envelope *ev;
 
-    ee = GNUNET_CONTAINER_multihashmap_get (op->spec->set->content->elements,
+    ee = GNUNET_CONTAINER_multihashmap_get (op->set->content->elements,
                                             hash);
     if (NULL != ee)
       if (GNUNET_YES == _GSS_is_element_of_operation (ee, op))
@@ -1868,9 +2076,12 @@ handle_p2p_offer (void *cls,
     ev = GNUNET_MQ_msg_header_extra (demands,
                                      sizeof (struct GNUNET_HashCode),
                                      GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND);
-    *(struct GNUNET_HashCode *) &demands[1] = *hash;
+    GNUNET_memcpy (&demands[1],
+                   hash,
+                   sizeof (struct GNUNET_HashCode));
     GNUNET_MQ_send (op->mq, ev);
   }
+  GNUNET_CADET_receive_done (op->channel);
 }
 
 
@@ -1880,16 +2091,22 @@ handle_p2p_offer (void *cls,
  * @param cls the union operation
  * @param mh the message
  */
-static void
-handle_p2p_done (void *cls,
-                 const struct GNUNET_MessageHeader *mh)
+void
+handle_union_p2p_done (void *cls,
+                       const struct GNUNET_MessageHeader *mh)
 {
   struct Operation *op = cls;
 
-  if (op->state->phase == PHASE_INVENTORY_PASSIVE)
+  if (GNUNET_SET_OPERATION_UNION != op->set->operation)
+  {
+    GNUNET_break_op (0);
+    fail_union_operation (op);
+    return;
+  }
+  switch (op->state->phase)
   {
+  case PHASE_INVENTORY_PASSIVE:
     /* We got all requests, but still have to send our elements in response. */
-
     op->state->phase = PHASE_FINISH_WAITING;
 
     LOG (GNUNET_ERROR_TYPE_DEBUG,
@@ -1903,11 +2120,10 @@ handle_p2p_done (void *cls,
      * all our demands are satisfied, so that the active
      * peer can quit if we gave him everything.
      */
+    GNUNET_CADET_receive_done (op->channel);
     maybe_finish (op);
     return;
-  }
-  if (op->state->phase == PHASE_INVENTORY_ACTIVE)
-  {
+  case PHASE_INVENTORY_ACTIVE:
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "got DONE (as active partner), waiting to finish\n");
     /* All demands of the other peer are satisfied,
@@ -1918,11 +2134,14 @@ handle_p2p_done (void *cls,
      * to the other peer once our demands are met.
      */
     op->state->phase = PHASE_FINISH_CLOSING;
+    GNUNET_CADET_receive_done (op->channel);
     maybe_finish (op);
     return;
+  default:
+    GNUNET_break_op (0);
+    fail_union_operation (op);
+    return;
   }
-  GNUNET_break_op (0);
-  fail_union_operation (op);
 }
 
 
@@ -1933,21 +2152,31 @@ handle_p2p_done (void *cls,
  * @param opaque_context message to be transmitted to the listener
  *        to convince him to accept, may be NULL
  */
-static void
+static struct OperationState *
 union_evaluate (struct Operation *op,
                 const struct GNUNET_MessageHeader *opaque_context)
 {
+  struct OperationState *state;
   struct GNUNET_MQ_Envelope *ev;
   struct OperationRequestMessage *msg;
 
-  GNUNET_assert (NULL == op->state);
-  op->state = GNUNET_new (struct OperationState);
-  op->state->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO);
+  ev = GNUNET_MQ_msg_nested_mh (msg,
+                                GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
+                                opaque_context);
+  if (NULL == ev)
+  {
+    /* the context message is too large */
+    GNUNET_break (0);
+    return NULL;
+  }
+  state = GNUNET_new (struct OperationState);
+  state->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32,
+                                                                 GNUNET_NO);
   /* copy the current generation's strata estimator for this operation */
-  op->state->se = strata_estimator_dup (op->spec->set->state->se);
+  state->se = strata_estimator_dup (op->set->state->se);
   /* we started the operation, thus we have to send the operation request */
-  op->state->phase = PHASE_EXPECT_SE;
-  op->state->salt_receive = op->state->salt_send = 42;
+  state->phase = PHASE_EXPECT_SE;
+  state->salt_receive = state->salt_send = 42; // FIXME?????
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Initiating union operation evaluation\n");
   GNUNET_STATISTICS_update (_GSS_statistics,
@@ -1958,16 +2187,6 @@ union_evaluate (struct Operation *op,
                             "# of initiated union operations",
                             1,
                             GNUNET_NO);
-  ev = GNUNET_MQ_msg_nested_mh (msg,
-                                GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
-                                opaque_context);
-  if (NULL == ev)
-  {
-    /* the context message is too large */
-    GNUNET_break (0);
-    GNUNET_SERVICE_client_drop (op->spec->set->client);
-    return;
-  }
   msg->operation = htonl (GNUNET_SET_OPERATION_UNION);
   GNUNET_MQ_send (op->mq,
                   ev);
@@ -1979,8 +2198,10 @@ union_evaluate (struct Operation *op,
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "sent op request without context message\n");
 
+  op->state = state;
   initialize_key_to_element (op);
-  op->state->initial_size = GNUNET_CONTAINER_multihashmap32_size (op->state->key_to_element);
+  state->initial_size = GNUNET_CONTAINER_multihashmap32_size (state->key_to_element);
+  return state;
 }
 
 
@@ -1990,13 +2211,19 @@ union_evaluate (struct Operation *op,
  *
  * @param op operation that will be accepted as a union operation
  */
-static void
+static struct OperationState *
 union_accept (struct Operation *op)
 {
+  struct OperationState *state;
+  const struct StrataEstimator *se;
+  struct GNUNET_MQ_Envelope *ev;
+  struct StrataEstimatorMessage *strata_msg;
+  char *buf;
+  size_t len;
+  uint16_t type;
+
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "accepting set union operation\n");
-  GNUNET_assert (NULL == op->state);
-
   GNUNET_STATISTICS_update (_GSS_statistics,
                             "# of accepted union operations",
                             1,
@@ -2006,14 +2233,37 @@ union_accept (struct Operation *op)
                             1,
                             GNUNET_NO);
 
-  op->state = GNUNET_new (struct OperationState);
-  op->state->se = strata_estimator_dup (op->spec->set->state->se);
-  op->state->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO);
-  op->state->salt_receive = op->state->salt_send = 42;
+  state = GNUNET_new (struct OperationState);
+  state->se = strata_estimator_dup (op->set->state->se);
+  state->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32,
+                                                                 GNUNET_NO);
+  state->salt_receive = state->salt_send = 42; // FIXME?????
+  op->state = state;
   initialize_key_to_element (op);
-  op->state->initial_size = GNUNET_CONTAINER_multihashmap32_size (op->state->key_to_element);
+  state->initial_size = GNUNET_CONTAINER_multihashmap32_size (state->key_to_element);
+
   /* kick off the operation */
-  send_strata_estimator (op);
+  se = state->se;
+  buf = GNUNET_malloc (se->strata_count * IBF_BUCKET_SIZE * se->ibf_size);
+  len = strata_estimator_write (se,
+                                buf);
+  if (len < se->strata_count * IBF_BUCKET_SIZE * se->ibf_size)
+    type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC;
+  else
+    type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE;
+  ev = GNUNET_MQ_msg_extra (strata_msg,
+                            len,
+                            type);
+  GNUNET_memcpy (&strata_msg[1],
+                 buf,
+                 len);
+  GNUNET_free (buf);
+  strata_msg->set_size
+    = GNUNET_htonll (GNUNET_CONTAINER_multihashmap_size (op->set->content->elements));
+  GNUNET_MQ_send (op->mq,
+                  ev);
+  state->phase = PHASE_EXPECT_IBF;
+  return state;
 }
 
 
@@ -2053,7 +2303,8 @@ union_set_create (void)
  * @param ee the element to add to the set
  */
 static void
-union_add (struct SetState *set_state, struct ElementEntry *ee)
+union_add (struct SetState *set_state,
+           struct ElementEntry *ee)
 {
   strata_estimator_insert (set_state->se,
                            get_ibf_key (&ee->element_hash));
@@ -2068,7 +2319,8 @@ union_add (struct SetState *set_state, struct ElementEntry *ee)
  * @param ee set element to remove
  */
 static void
-union_remove (struct SetState *set_state, struct ElementEntry *ee)
+union_remove (struct SetState *set_state,
+              struct ElementEntry *ee)
 {
   strata_estimator_remove (set_state->se,
                            get_ibf_key (&ee->element_hash));
@@ -2093,113 +2345,35 @@ union_set_destroy (struct SetState *set_state)
 
 
 /**
- * Dispatch messages for a union operation.
+ * Copy union-specific set state.
  *
- * @param op the state of the union evaluate operation
- * @param mh the received message
- * @return #GNUNET_SYSERR if the tunnel should be disconnected,
- *         #GNUNET_OK otherwise
+ * @param state source state for copying the union state
+ * @return a copy of the union-specific set state
  */
-int
-union_handle_p2p_message (struct Operation *op,
-                          const struct GNUNET_MessageHeader *mh)
+static struct SetState *
+union_copy_state (struct SetState *state)
 {
-  //LOG (GNUNET_ERROR_TYPE_DEBUG,
-  //            "received p2p message (t: %u, s: %u)\n",
-  //            ntohs (mh->type),
-  //            ntohs (mh->size));
-  switch (ntohs (mh->type))
-  {
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF:
-      return handle_p2p_ibf (op, mh);
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE:
-      return handle_p2p_strata_estimator (op, mh, GNUNET_NO);
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC:
-      return handle_p2p_strata_estimator (op, mh, GNUNET_YES);
-    case GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS:
-      handle_p2p_elements (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT:
-      handle_p2p_full_element (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY:
-      handle_p2p_inquiry (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE:
-      handle_p2p_done (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER:
-      handle_p2p_offer (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND:
-      handle_p2p_demand (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE:
-      handle_p2p_full_done (op, mh);
-      break;
-    case GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL:
-      handle_p2p_request_full (op, mh);
-      break;
-    default:
-      /* Something wrong with cadet's message handlers? */
-      GNUNET_assert (0);
-  }
-  return GNUNET_OK;
-}
+  struct SetState *new_state;
 
+  GNUNET_assert ( (NULL != state) &&
+                  (NULL != state->se) );
+  new_state = GNUNET_new (struct SetState);
+  new_state->se = strata_estimator_dup (state->se);
 
-/**
- * Handler for peer-disconnects, notifies the client
- * about the aborted operation in case the op was not concluded.
- *
- * @param op the destroyed operation
- */
-static void
-union_peer_disconnect (struct Operation *op)
-{
-  if (PHASE_DONE != op->state->phase)
-  {
-    struct GNUNET_MQ_Envelope *ev;
-    struct GNUNET_SET_ResultMessage *msg;
-
-    ev = GNUNET_MQ_msg (msg,
-                        GNUNET_MESSAGE_TYPE_SET_RESULT);
-    msg->request_id = htonl (op->spec->client_request_id);
-    msg->result_status = htons (GNUNET_SET_STATUS_FAILURE);
-    msg->element_type = htons (0);
-    GNUNET_MQ_send (op->spec->set->client_mq,
-                    ev);
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         "other peer disconnected prematurely, phase %u\n",
-         op->state->phase);
-    _GSS_operation_destroy (op,
-                            GNUNET_YES);
-    return;
-  }
-  // else: the session has already been concluded
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "other peer disconnected (finished)\n");
-  if (GNUNET_NO == op->state->client_done_sent)
-    send_done_and_destroy (op);
+  return new_state;
 }
 
 
 /**
- * Copy union-specific set state.
+ * Handle case where channel went down for an operation.
  *
- * @param set source set for copying the union state
- * @return a copy of the union-specific set state
+ * @param op operation that lost the channel
  */
-static struct SetState *
-union_copy_state (struct Set *set)
+static void
+union_channel_death (struct Operation *op)
 {
-  struct SetState *new_state;
-
-  new_state = GNUNET_new (struct SetState);
-  GNUNET_assert ( (NULL != set->state) && (NULL != set->state->se) );
-  new_state->se = strata_estimator_dup (set->state->se);
-
-  return new_state;
+  _GSS_operation_destroy (op,
+                          GNUNET_YES);
 }
 
 
@@ -2214,15 +2388,14 @@ _GSS_union_vt ()
 {
   static const struct SetVT union_vt = {
     .create = &union_set_create,
-    .msg_handler = &union_handle_p2p_message,
     .add = &union_add,
     .remove = &union_remove,
     .destroy_set = &union_set_destroy,
     .evaluate = &union_evaluate,
     .accept = &union_accept,
-    .peer_disconnect = &union_peer_disconnect,
     .cancel = &union_op_cancel,
     .copy_state = &union_copy_state,
+    .channel_death = &union_channel_death
   };
 
   return &union_vt;
diff --git a/src/set/gnunet-service-set_union.h b/src/set/gnunet-service-set_union.h
new file mode 100644
index 000000000..cbf60bcbc
--- /dev/null
+++ b/src/set/gnunet-service-set_union.h
@@ -0,0 +1,239 @@
+
+/*
+      This file is part of GNUnet
+      Copyright (C) 2013-2017 GNUnet e.V.
+
+      GNUnet is free software; you can redistribute it and/or modify
+      it under the terms of the GNU General Public License as published
+      by the Free Software Foundation; either version 3, or (at your
+      option) any later version.
+
+      GNUnet is distributed in the hope that it will be useful, but
+      WITHOUT ANY WARRANTY; without even the implied warranty of
+      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+      General Public License for more details.
+
+      You should have received a copy of the GNU General Public License
+      along with GNUnet; see the file COPYING.  If not, write to the
+      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+      Boston, MA 02110-1301, USA.
+*/
+/**
+ * @file set/gnunet-service-set_union.h
+ * @brief two-peer set operations
+ * @author Florian Dold
+ * @author Christian Grothoff
+ */
+#ifndef GNUNET_SERVICE_SET_UNION_H
+#define GNUNET_SERVICE_SET_UNION_H
+
+#include "gnunet-service-set.h"
+#include "gnunet-service-set_protocol.h"
+
+
+/**
+ * Handle a strata estimator from a remote peer
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+int
+check_union_p2p_strata_estimator (void *cls,
+                                  const struct StrataEstimatorMessage *msg);
+
+
+/**
+ * Handle a strata estimator from a remote peer
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+void
+handle_union_p2p_strata_estimator (void *cls,
+                                   const struct StrataEstimatorMessage *msg);
+
+
+/**
+ * Check an IBF message from a remote peer.
+ *
+ * Reassemble the IBF from multiple pieces, and
+ * process the whole IBF once possible.
+ *
+ * @param cls the union operation
+ * @param msg the header of the message
+ * @return #GNUNET_OK if @a msg is well-formed
+ */
+int
+check_union_p2p_ibf (void *cls,
+                     const struct IBFMessage *msg);
+
+
+/**
+ * Handle an IBF message from a remote peer.
+ *
+ * Reassemble the IBF from multiple pieces, and
+ * process the whole IBF once possible.
+ *
+ * @param cls the union operation
+ * @param msg the header of the message
+ */
+void
+handle_union_p2p_ibf (void *cls,
+                      const struct IBFMessage *msg);
+
+
+/**
+ * Check an element message from a remote peer.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+int
+check_union_p2p_elements (void *cls,
+                          const struct GNUNET_SET_ElementMessage *emsg);
+
+
+/**
+ * Handle an element message from a remote peer.
+ * Sent by the other peer either because we decoded an IBF and placed a demand,
+ * or because the other peer switched to full set transmission.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+void
+handle_union_p2p_elements (void *cls,
+                           const struct GNUNET_SET_ElementMessage *emsg);
+
+
+/**
+ * Check a full element message from a remote peer.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+int
+check_union_p2p_full_element (void *cls,
+                              const struct GNUNET_SET_ElementMessage *emsg);
+
+
+/**
+ * Handle an element message from a remote peer.
+ *
+ * @param cls the union operation
+ * @param emsg the message
+ */
+void
+handle_union_p2p_full_element (void *cls,
+                               const struct GNUNET_SET_ElementMessage *emsg);
+
+
+/**
+ * Send offers (for GNUNET_Hash-es) in response
+ * to inquiries (for IBF_Key-s).
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+int
+check_union_p2p_inquiry (void *cls,
+                         const struct InquiryMessage *msg);
+
+
+/**
+ * Send offers (for GNUNET_Hash-es) in response
+ * to inquiries (for IBF_Key-s).
+ *
+ * @param cls the union operation
+ * @param msg the message
+ */
+void
+handle_union_p2p_inquiry (void *cls,
+                          const struct InquiryMessage *msg);
+
+
+
+/**
+ * Handle a request for full set transmission.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ */
+void
+handle_union_p2p_request_full (void *cls,
+                               const struct GNUNET_MessageHeader *mh);
+
+
+
+/**
+ * Handle a "full done" message.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ */
+void
+handle_union_p2p_full_done (void *cls,
+                            const struct GNUNET_MessageHeader *mh);
+
+
+/**
+ * Check a demand by the other peer for elements based on a list
+ * of `struct GNUNET_HashCode`s.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ * @return #GNUNET_OK if @a mh is well-formed
+ */
+int
+check_union_p2p_demand (void *cls,
+                        const struct GNUNET_MessageHeader *mh);
+
+
+/**
+ * Handle a demand by the other peer for elements based on a list
+ * of `struct GNUNET_HashCode`s.
+ *
+ * @parem cls closure, a set union operation
+ * @param mh the demand message
+ */
+void
+handle_union_p2p_demand (void *cls,
+                         const struct GNUNET_MessageHeader *mh);
+
+
+/**
+ * Check offer (of `struct GNUNET_HashCode`s).
+ *
+ * @param cls the union operation
+ * @param mh the message
+ * @return #GNUNET_OK if @a mh is well-formed
+ */
+int
+check_union_p2p_offer (void *cls,
+                       const struct GNUNET_MessageHeader *mh);
+
+
+/**
+ * Handle offers (of `struct GNUNET_HashCode`s) and
+ * respond with demands (of `struct GNUNET_HashCode`s).
+ *
+ * @param cls the union operation
+ * @param mh the message
+ */
+void
+handle_union_p2p_offer (void *cls,
+                        const struct GNUNET_MessageHeader *mh);
+
+
+/**
+ * Handle a done message from a remote peer
+ *
+ * @param cls the union operation
+ * @param mh the message
+ */
+void
+handle_union_p2p_done (void *cls,
+                       const struct GNUNET_MessageHeader *mh);
+
+
+#endif
diff --git a/src/set/gnunet-set-ibf-profiler.c b/src/set/gnunet-set-ibf-profiler.c
index 8d832e358..ac86a900d 100644
--- a/src/set/gnunet-set-ibf-profiler.c
+++ b/src/set/gnunet-set-ibf-profiler.c
@@ -244,24 +244,41 @@ run (void *cls,
 int
 main (int argc, char **argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'A', "asize", NULL,
-     gettext_noop ("number of element in set A-B"), 1,
-     &GNUNET_GETOPT_set_uint, &asize},
-    {'B', "bsize", NULL,
-     gettext_noop ("number of element in set B-A"), 1,
-     &GNUNET_GETOPT_set_uint, &bsize},
-    {'C', "csize", NULL,
-     gettext_noop ("number of common elements in A and B"), 1,
-     &GNUNET_GETOPT_set_uint, &csize},
-    {'k', "hash-num", NULL,
-     gettext_noop ("hash num"), 1,
-     &GNUNET_GETOPT_set_uint, &hash_num},
-    {'s', "ibf-size", NULL,
-     gettext_noop ("ibf size"), 1,
-     &GNUNET_GETOPT_set_uint, &ibf_size},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('A',
+                                   "asize",
+                                   NULL,
+                                   gettext_noop ("number of element in set A-B"),
+                                   &asize),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('B',
+                                   "bsize",
+                                   NULL,
+                                   gettext_noop ("number of element in set B-A"),
+                                   &bsize),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('C',
+                                   "csize",
+                                   NULL,
+                                   gettext_noop ("number of common elements in A and B"),
+                                   &csize),
+    
+    GNUNET_GETOPT_OPTION_SET_UINT ('k',
+                                   "hash-num",
+                                   NULL,
+                                   gettext_noop ("hash num"),
+                                   &hash_num),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('s',
+                                   "ibf-size",
+                                   NULL,
+                                   gettext_noop ("ibf size"),
+                                   &ibf_size),
+
     GNUNET_GETOPT_OPTION_END
   };
+
   GNUNET_PROGRAM_run2 (argc, argv, "gnunet-consensus-ibf",
                       "help",
                       options, &run, NULL, GNUNET_YES);
diff --git a/src/set/gnunet-set-profiler.c b/src/set/gnunet-set-profiler.c
index d83e034a6..349bce6ea 100644
--- a/src/set/gnunet-set-profiler.c
+++ b/src/set/gnunet-set-profiler.c
@@ -59,8 +59,9 @@ static struct GNUNET_PeerIdentity local_peer;
 static struct GNUNET_SET_ListenHandle *set_listener;
 
 static int byzantine;
-static int force_delta;
-static int force_full;
+static unsigned int force_delta;
+static unsigned int force_full;
+static unsigned int element_size = 32;
 
 /**
  * Handle to the statistics service.
@@ -90,7 +91,7 @@ map_remove_iterator (void *cls,
 
   GNUNET_assert (NULL != key);
 
-  ret = GNUNET_CONTAINER_multihashmap_remove (m, key, NULL);
+  ret = GNUNET_CONTAINER_multihashmap_remove_all (m, key);
   if (GNUNET_OK != ret)
     printf ("spurious element\n");
   return GNUNET_YES;
@@ -196,7 +197,7 @@ set_result_cb (void *cls,
       GNUNET_assert (0);
   }
 
-  if (element->size != sizeof (struct GNUNET_HashCode))
+  if (element->size != element_size)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "wrong element size: %u, expected %u\n",
@@ -208,8 +209,10 @@ set_result_cb (void *cls,
   GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s: got element (%s)\n",
               info->id, GNUNET_h2s (element->data));
   GNUNET_assert (NULL != element->data);
+  struct GNUNET_HashCode data_hash;
+  GNUNET_CRYPTO_hash (element->data, element_size, &data_hash);
   GNUNET_CONTAINER_multihashmap_put (info->received,
-                                     element->data, NULL,
+                                     &data_hash, NULL,
                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
 }
 
@@ -261,16 +264,12 @@ set_insert_iterator (void *cls,
                      void *value)
 {
   struct GNUNET_SET_Handle *set = cls;
-  struct GNUNET_SET_Element *el;
-
-  el = GNUNET_malloc (sizeof (struct GNUNET_SET_Element) +
-                      sizeof (struct GNUNET_HashCode));
-  el->element_type = 0;
-  GNUNET_memcpy (&el[1], key, sizeof *key);
-  el->data = &el[1];
-  el->size = sizeof *key;
-  GNUNET_SET_add_element (set, el, NULL, NULL);
-  GNUNET_free (el);
+  struct GNUNET_SET_Element el;
+
+  el.element_type = 0;
+  el.data = value;
+  el.size = element_size;
+  GNUNET_SET_add_element (set, &el, NULL, NULL);
   return GNUNET_YES;
 }
 
@@ -322,6 +321,8 @@ run (void *cls,
 
   config = cfg;
 
+  GNUNET_assert (element_size > 0);
+
   if (GNUNET_OK != GNUNET_CRYPTO_get_peer_identity (cfg, &local_peer))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not retrieve host identity\n");
@@ -345,22 +346,28 @@ run (void *cls,
 
   for (i = 0; i < num_a; i++)
   {
-    GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &hash);
-    GNUNET_CONTAINER_multihashmap_put (info1.sent, &hash, NULL,
+    char *data = GNUNET_malloc (element_size);
+    GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
+    GNUNET_CRYPTO_hash (data, element_size, &hash);
+    GNUNET_CONTAINER_multihashmap_put (info1.sent, &hash, data,
                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
   }
 
   for (i = 0; i < num_b; i++)
   {
-    GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &hash);
-    GNUNET_CONTAINER_multihashmap_put (info2.sent, &hash, NULL,
+    char *data = GNUNET_malloc (element_size);
+    GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
+    GNUNET_CRYPTO_hash (data, element_size, &hash);
+    GNUNET_CONTAINER_multihashmap_put (info2.sent, &hash, data,
                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
   }
 
   for (i = 0; i < num_c; i++)
   {
-    GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &hash);
-    GNUNET_CONTAINER_multihashmap_put (common_sent, &hash, NULL,
+    char *data = GNUNET_malloc (element_size);
+    GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
+    GNUNET_CRYPTO_hash (data, element_size, &hash);
+    GNUNET_CONTAINER_multihashmap_put (common_sent, &hash, data,
                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
   }
 
@@ -419,31 +426,60 @@ pre_run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char **argv)
 {
-   static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-      { 'A', "num-first", NULL,
-        gettext_noop ("number of values"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_a },
-      { 'B', "num-second", NULL,
-        gettext_noop ("number of values"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_b },
-      { 'b', "byzantine", NULL,
-        gettext_noop ("use byzantine mode"),
-        GNUNET_NO, &GNUNET_GETOPT_set_one, &byzantine },
-      { 'f', "force-full", NULL,
-        gettext_noop ("force sending full set"),
-        GNUNET_NO, &GNUNET_GETOPT_set_uint, &force_full },
-      { 'd', "force-delta", NULL,
-        gettext_noop ("number delta operation"),
-        GNUNET_NO, &GNUNET_GETOPT_set_uint, &force_delta },
-      { 'C', "num-common", NULL,
-        gettext_noop ("number of values"),
-        GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_c },
-      { 'x', "operation", NULL,
-        gettext_noop ("operation to execute"),
-        GNUNET_YES, &GNUNET_GETOPT_set_string, &op_str },
-      { 's', "statistics", NULL,
-        gettext_noop ("write statistics to file"),
-        GNUNET_YES, &GNUNET_GETOPT_set_filename, &statistics_filename },
+   struct GNUNET_GETOPT_CommandLineOption options[] = {
+      GNUNET_GETOPT_OPTION_SET_UINT ('A',
+                                     "num-first",
+                                     NULL,
+                                     gettext_noop ("number of values"),
+                                     &num_a),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('B',
+                                     "num-second",
+                                     NULL,
+                                     gettext_noop ("number of values"),
+                                     &num_b),
+
+      GNUNET_GETOPT_OPTION_SET_ONE ('b',
+                                    "byzantine",
+                                    gettext_noop ("use byzantine mode"),
+                                    &byzantine),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('f',
+                                     "force-full",
+                                     NULL,
+                                     gettext_noop ("force sending full set"),
+                                     &force_full),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('d',
+                                     "force-delta",
+                                     NULL,
+                                     gettext_noop ("number delta operation"),
+                                     &force_delta),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('C',
+                                     "num-common",
+                                     NULL,
+                                     gettext_noop ("number of values"),
+                                     &num_c),
+
+      GNUNET_GETOPT_OPTION_STRING ('x',
+                                   "operation",
+                                   NULL,
+                                   gettext_noop ("operation to execute"),
+                                   &op_str),
+
+      GNUNET_GETOPT_OPTION_SET_UINT ('w',
+                                     "element-size",
+                                     NULL,
+                                     gettext_noop ("element size"),
+                                     &element_size),
+
+      GNUNET_GETOPT_OPTION_FILENAME ('s',
+                                     "statistics",
+                                     "FILENAME",
+                                     gettext_noop ("write statistics to file"),
+                                     &statistics_filename),
+
       GNUNET_GETOPT_OPTION_END
   };
   GNUNET_PROGRAM_run2 (argc, argv, "gnunet-set-profiler",
diff --git a/src/set/set_api.c b/src/set/set_api.c
index 5b5b1b8ee..f5c43a9a7 100644
--- a/src/set/set_api.c
+++ b/src/set/set_api.c
@@ -76,6 +76,8 @@ struct GNUNET_SET_Handle
 
   /**
    * Should the set be destroyed once all operations are gone?
+   * #GNUNET_SYSERR if #GNUNET_SET_destroy() must raise this flag,
+   * #GNUNET_YES if #GNUNET_SET_destroy() did raise this flag.
    */
   int destroy_requested;
 
@@ -308,6 +310,8 @@ handle_iter_element (void *cls,
   struct GNUNET_MQ_Envelope *ev;
   uint16_t msize;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received element in set iteration\n");
   msize = ntohs (msg->header.size);
   if (set->iteration_id != ntohs (msg->iteration_id))
   {
@@ -344,11 +348,24 @@ handle_iter_done (void *cls,
   GNUNET_SET_ElementIterator iter = set->iterator;
 
   if (NULL == iter)
+  {
+    /* FIXME: if this is true, could cancel+start a fresh one
+       cause elements to go to the wrong iteration? */
+    LOG (GNUNET_ERROR_TYPE_INFO,
+         "Service completed set iteration that was already cancelled\n");
     return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Set iteration completed\n");
+  set->destroy_requested = GNUNET_SYSERR;
   set->iterator = NULL;
   set->iteration_id++;
   iter (set->iterator_cls,
         NULL);
+  if (GNUNET_SYSERR == set->destroy_requested)
+    set->destroy_requested = GNUNET_NO;
+  if (GNUNET_YES == set->destroy_requested)
+    GNUNET_SET_destroy (set);
 }
 
 
@@ -385,7 +402,7 @@ handle_result (void *cls,
   int destroy_set;
 
   GNUNET_assert (NULL != set->mq);
-  result_status = ntohs (msg->result_status);
+  result_status = (enum GNUNET_SET_Status) ntohs (msg->result_status);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Got result message with status %d\n",
        result_status);
@@ -500,6 +517,8 @@ GNUNET_SET_operation_cancel (struct GNUNET_SET_OperationHandle *oh)
   struct GNUNET_SET_CancelMessage *m;
   struct GNUNET_MQ_Envelope *mqm;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Cancelling SET operation\n");
   if (NULL != set)
   {
     mqm = GNUNET_MQ_msg (m, GNUNET_MESSAGE_TYPE_SET_CANCEL);
@@ -553,6 +572,9 @@ handle_client_set_error (void *cls,
 }
 
 
+/**
+ * FIXME.
+ */
 static struct GNUNET_SET_Handle *
 create_internal (const struct GNUNET_CONFIGURATION_Handle *cfg,
                  enum GNUNET_SET_OperationType op,
@@ -611,7 +633,8 @@ create_internal (const struct GNUNET_CONFIGURATION_Handle *cfg,
                          GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_CONNECT);
     copy_msg->cookie = *cookie;
   }
-  GNUNET_MQ_send (set->mq, mqm);
+  GNUNET_MQ_send (set->mq,
+                  mqm);
   return set;
 }
 
@@ -631,7 +654,16 @@ struct GNUNET_SET_Handle *
 GNUNET_SET_create (const struct GNUNET_CONFIGURATION_Handle *cfg,
                    enum GNUNET_SET_OperationType op)
 {
-  return create_internal (cfg, op, NULL);
+  struct GNUNET_SET_Handle *set;
+
+  set = create_internal (cfg,
+                          op,
+                          NULL);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating set %p for operation %d\n",
+       set,
+       op);
+  return set;
 }
 
 
@@ -657,8 +689,10 @@ GNUNET_SET_add_element (struct GNUNET_SET_Handle *set,
   struct GNUNET_MQ_Envelope *mqm;
   struct GNUNET_SET_ElementMessage *msg;
 
-  LOG (GNUNET_ERROR_TYPE_INFO, "adding element of type %u\n", (unsigned) element->element_type);
-
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "adding element of type %u to set %p\n",
+       (unsigned int) element->element_type,
+       set);
   if (GNUNET_YES == set->invalid)
   {
     if (NULL != cont)
@@ -701,6 +735,9 @@ GNUNET_SET_remove_element (struct GNUNET_SET_Handle *set,
   struct GNUNET_MQ_Envelope *mqm;
   struct GNUNET_SET_ElementMessage *msg;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Removing element from set %p\n",
+       set);
   if (GNUNET_YES == set->invalid)
   {
     if (NULL != cont)
@@ -733,8 +770,9 @@ GNUNET_SET_destroy (struct GNUNET_SET_Handle *set)
   /* destroying set while iterator is active is currently
      not supported; we should expand the API to allow
      clients to explicitly cancel the iteration! */
-  GNUNET_assert (NULL == set->iterator);
-  if (NULL != set->ops_head)
+  if ( (NULL != set->ops_head) ||
+       (NULL != set->iterator) ||
+       (GNUNET_SYSERR == set->destroy_requested) )
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Set operations are pending, delaying set destruction\n");
@@ -807,7 +845,7 @@ GNUNET_SET_prepare (const struct GNUNET_PeerIdentity *other_peer,
         msg->force_delta = GNUNET_YES;
         break;
       default:
-        LOG (GNUNET_ERROR_TYPE_ERROR, 
+        LOG (GNUNET_ERROR_TYPE_ERROR,
              "Option with type %d not recognized\n", (int) opt->type);
     }
   }
@@ -870,7 +908,8 @@ handle_request (void *cls,
   struct GNUNET_SET_RejectMessage *rmsg;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Processing incoming operation request\n");
+       "Processing incoming operation request with id %u\n",
+       ntohl (msg->accept_id));
   /* we got another valid request => reset the backoff */
   lh->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS;
   req.accept_id = ntohl (msg->accept_id);
@@ -884,7 +923,8 @@ handle_request (void *cls,
   if (GNUNET_YES == req.accepted)
     return; /* the accept-case is handled in #GNUNET_SET_accept() */
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Rejecting request\n");
+       "Rejected request %u\n",
+       ntohl (msg->accept_id));
   mqm = GNUNET_MQ_msg (rmsg,
                        GNUNET_MESSAGE_TYPE_SET_REJECT);
   rmsg->accept_reject_id = msg->accept_id;
@@ -974,6 +1014,9 @@ GNUNET_SET_listen (const struct GNUNET_CONFIGURATION_Handle *cfg,
 {
   struct GNUNET_SET_ListenHandle *lh;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Starting listener for app %s\n",
+       GNUNET_h2s (app_id));
   lh = GNUNET_new (struct GNUNET_SET_ListenHandle);
   lh->listen_cb = listen_cb;
   lh->listen_cls = listen_cls;
@@ -1000,7 +1043,8 @@ void
 GNUNET_SET_listen_cancel (struct GNUNET_SET_ListenHandle *lh)
 {
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Canceling listener\n");
+       "Canceling listener %s\n",
+       GNUNET_h2s (&lh->app_id));
   if (NULL != lh->mq)
   {
     GNUNET_MQ_destroy (lh->mq);
@@ -1042,10 +1086,12 @@ GNUNET_SET_accept (struct GNUNET_SET_Request *request,
 
   GNUNET_assert (GNUNET_NO == request->accepted);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Client accepts set operation (%d)\n",
-       result_mode);
+       "Client accepts set operation (%d) with id %u\n",
+       result_mode,
+       request->accept_id);
   request->accepted = GNUNET_YES;
-  mqm = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SET_ACCEPT);
+  mqm = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_SET_ACCEPT);
   msg->accept_reject_id = htonl (request->accept_id);
   msg->result_mode = htonl (result_mode);
   oh = GNUNET_new (struct GNUNET_SET_OperationHandle);
@@ -1143,6 +1189,8 @@ GNUNET_SET_copy_lazy (struct GNUNET_SET_Handle *set,
   struct GNUNET_MQ_Envelope *ev;
   struct SetCopyRequest *req;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating lazy copy of set\n");
   ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_PREPARE);
   GNUNET_MQ_send (set->mq, ev);
 
diff --git a/src/set/test_set_api.c b/src/set/test_set_api.c
index dd3f004f2..ca7d8a4e2 100644
--- a/src/set/test_set_api.c
+++ b/src/set/test_set_api.c
@@ -116,6 +116,7 @@ result_cb_set2 (void *cls,
     oh2 = NULL;
     fprintf (stderr,
              "set 2: received failure status\n");
+    GNUNET_SCHEDULER_shutdown ();
     ret = 1;
     break;
   case GNUNET_SET_STATUS_DONE:
@@ -147,8 +148,6 @@ listen_cb (void *cls,
   GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "listen cb called\n");
-  GNUNET_SET_listen_cancel (listen_handle);
-  listen_handle = NULL;
   oh2 = GNUNET_SET_accept (request,
                            GNUNET_SET_RESULT_ADDED,
                            (struct GNUNET_SET_Option[]) { 0 },
@@ -200,19 +199,25 @@ init_set2 (void *cls)
 {
   struct GNUNET_SET_Element element;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initializing set 2\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "initializing set 2\n");
 
   element.element_type = 0;
-
   element.data = "hello";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set2, &element, NULL, NULL);
+  GNUNET_SET_add_element (set2,
+                          &element,
+                          NULL, NULL);
   element.data = "quux";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set2, &element, NULL, NULL);
+  GNUNET_SET_add_element (set2,
+                          &element,
+                          NULL, NULL);
   element.data = "baz";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set2, &element, &start, NULL);
+  GNUNET_SET_add_element (set2,
+                          &element,
+                          &start, NULL);
 }
 
 
@@ -225,14 +230,17 @@ init_set1 (void)
   struct GNUNET_SET_Element element;
 
   element.element_type = 0;
-
   element.data = "hello";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set1, &element, NULL, NULL);
+  GNUNET_SET_add_element (set1,
+                          &element,
+                          NULL, NULL);
   element.data = "bar";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (set1, &element, init_set2, NULL);
-
+  GNUNET_SET_add_element (set1,
+                          &element,
+                          &init_set2,
+                          NULL);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "initialized set 1\n");
 }
@@ -242,10 +250,15 @@ static int
 iter_cb (void *cls,
          const struct GNUNET_SET_Element *element)
 {
+  struct GNUNET_SET_Handle *set = cls;
+
   if (NULL == element)
   {
     GNUNET_assert (3 == iter_count);
-    GNUNET_SET_destroy (cls);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Iteration finished, destroying set %p\n",
+                set);
+    GNUNET_SET_destroy (set);
     return GNUNET_YES;
   }
   iter_count++;
@@ -262,21 +275,31 @@ test_iter ()
   struct GNUNET_SET_Element element;
   struct GNUNET_SET_Handle *iter_set;
 
-  iter_set = GNUNET_SET_create (config, GNUNET_SET_OPERATION_UNION);
-
+  iter_set = GNUNET_SET_create (config,
+                                GNUNET_SET_OPERATION_UNION);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Testing iteration over 3 elements on set %p\n",
+              iter_set);
   element.element_type = 0;
 
   element.data = "hello";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
+  GNUNET_SET_add_element (iter_set,
+                          &element,
+                          NULL, NULL);
   element.data = "bar";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
+  GNUNET_SET_add_element (iter_set,
+                          &element,
+                          NULL, NULL);
   element.data = "quux";
   element.size = strlen(element.data);
-  GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
-
-  GNUNET_SET_iterate (iter_set, iter_cb, iter_set);
+  GNUNET_SET_add_element (iter_set,
+                          &element,
+                          NULL, NULL);
+  GNUNET_SET_iterate (iter_set,
+                      &iter_cb,
+                      iter_set);
 }
 
 
@@ -372,12 +395,20 @@ run (void *cls,
               GNUNET_i2s (&local_id));
   test_iter ();
 
-  set1 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
-  set2 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
+  set1 = GNUNET_SET_create (cfg,
+                            GNUNET_SET_OPERATION_UNION);
+  set2 = GNUNET_SET_create (cfg,
+                            GNUNET_SET_OPERATION_UNION);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Created sets %p and %p for union operation\n",
+              set1,
+              set2);
   GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
                                     &app_id);
 
-  ///* test if canceling an uncommited request works! */
+  /* test if canceling an uncommited request works! */
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Launching and instantly stopping set operation\n");
   my_oh = GNUNET_SET_prepare (&local_id,
                               &app_id,
                               NULL,
@@ -385,7 +416,6 @@ run (void *cls,
                               (struct GNUNET_SET_Option[]) { 0 },
                               NULL,
                               NULL);
-
   GNUNET_SET_operation_cancel (my_oh);
 
   /* test the real set reconciliation */
diff --git a/src/set/test_set_union_copy.c b/src/set/test_set_union_copy.c
index c887a8958..242b9f2f2 100644
--- a/src/set/test_set_union_copy.c
+++ b/src/set/test_set_union_copy.c
@@ -122,6 +122,7 @@ check_count_iter (void *cls,
       return GNUNET_NO;
     }
     ci_cls->cont (ci_cls->cont_cls);
+    GNUNET_free (ci_cls);
     return GNUNET_NO;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -264,7 +265,8 @@ run (void *cls,
   GNUNET_TESTING_peer_get_identity (peer,
                                     &local_id);
 
-  set1 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
+  set1 = GNUNET_SET_create (cfg,
+                            GNUNET_SET_OPERATION_UNION);
   add_element_str (set1,
                    "333");
   add_element_str (set1,
diff --git a/src/social/.gitignore b/src/social/.gitignore
index 41954615f..875aa1105 100644
--- a/src/social/.gitignore
+++ b/src/social/.gitignore
@@ -1,2 +1,3 @@
 gnunet-social
 gnunet-service-social
+test_social
diff --git a/src/social/gnunet-social.c b/src/social/gnunet-social.c
index 1da51243e..4a46fdc99 100644
--- a/src/social/gnunet-social.c
+++ b/src/social/gnunet-social.c
@@ -67,10 +67,10 @@ static int op_guest_leave;
 static int op_guest_talk;
 
 /** --replay */
-static char *op_replay;
+static int op_replay;
 
 /** --replay-latest */
-static char *op_replay_latest;
+static int op_replay_latest;
 
 /** --look-at */
 static int op_look_at;
@@ -116,13 +116,13 @@ static char *opt_data;
 static char *opt_name;
 
 /** --start */
-static uint64_t opt_start;
+static unsigned long long opt_start;
 
 /** --until */
-static uint64_t opt_until;
+static unsigned long long opt_until;
 
 /** --limit */
-static int opt_limit;
+static unsigned long long opt_limit;
 
 
 /* global vars */
@@ -563,7 +563,7 @@ slicer_recv_method (void *cls,
               "%s (flags: %x)\n",
               message_id, method_name, ntohl (meth->flags));
   /* routing header is missing, so we just print double newline */
-  printf(".\n\n");
+  printf("\n");
   /* we output . instead of | to indicate that this is not proper PSYC syntax */
   /* FIXME: use libpsyc here */
 }
@@ -588,10 +588,11 @@ slicer_recv_modifier (void *cls,
               "Received modifier for message ID %" PRIu64 ":\n"
               "%c%s: %.*s (size: %u)\n",
               message_id, oper, name, value_size, (const char *) value, value_size);
-#endif
+#else
   /* obviously not binary safe */
   printf("%c%s\t%.*s\n",
               oper, name, value_size, (const char *) value);
+#endif
 }
 
 
@@ -611,10 +612,11 @@ slicer_recv_data (void *cls,
               "Received data for message ID %" PRIu64 ":\n"
               "%.*s\n",
               message_id, data_size, (const char *) data);
-#endif
+#else
   /* obviously not binary safe */
   printf("%s\n%.*s\n",
               method_received, data_size, (const char *) data);
+#endif
 }
 
 
@@ -1016,6 +1018,7 @@ app_connected (void *cls)
       guest_enter (&place_pub_key, &peer);
     }
   }
+  printf(".\n");
 }
 
 
@@ -1161,6 +1164,7 @@ run (void *cls, char *const *args, const char *cfgfile,
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 _("--place missing or invalid.\n"));
+    /* FIXME: why does it segfault here? */
     exit_fail ();
     return;
   }
@@ -1195,7 +1199,7 @@ int
 main (int argc, char *const *argv)
 {
   int res;
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
     /*
      * gnunet program options in addition to the ones below:
      *
@@ -1208,120 +1212,165 @@ main (int argc, char *const *argv)
 
     /* operations */
 
-    { 'A', "host-assign", NULL,
-      gettext_noop ("assign --name in state to --data"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_host_assign },
-
-    { 'B', "guest-leave", NULL,
-      gettext_noop ("say good-bye and leave somebody else's place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_guest_leave },
-
-    { 'C', "host-enter", NULL,
-      gettext_noop ("create a place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_host_enter },
-
-    { 'D', "host-leave", NULL,
-      gettext_noop ("destroy a place we were hosting"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_host_leave },
-
-    { 'E', "guest-enter", NULL,
-      gettext_noop ("enter somebody else's place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_guest_enter },
-
-    { 'F', "look-for", NULL,
-      gettext_noop ("find state matching name prefix"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_look_for },
-
-    { 'H', "replay-latest", NULL,
-      gettext_noop ("replay history of messages up to the given --limit"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_replay_latest },
-
-    { 'N', "host-reconnect", NULL,
-      gettext_noop ("reconnect to a previously created place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_host_reconnect },
-
-    { 'P', "host-announce", NULL,
-      gettext_noop ("publish something to a place we are hosting"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_host_announce },
-
-    { 'R', "guest-reconnect", NULL,
-      gettext_noop ("reconnect to a previously entered place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_guest_reconnect },
-
-    { 'S', "look-at", NULL,
-      gettext_noop ("search for state matching exact name"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_look_at },
-
-    { 'T', "guest-talk", NULL,
-      gettext_noop ("submit something to somebody's place"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_guest_talk },
-
-    { 'U', "status", NULL,
-      gettext_noop ("list of egos and subscribed places"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_status },
-
-    { 'X', "replay", NULL,
-      gettext_noop ("extract and replay history between message IDs --start and --until"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &op_replay },
+    GNUNET_GETOPT_OPTION_SET_ONE ('A',
+                                  "host-assign",
+                                  gettext_noop ("assign --name in state to --data"),
+                                  &op_host_assign),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('B',
+                                  "guest-leave",
+                                  gettext_noop ("say good-bye and leave somebody else's place"),
+                                  &op_guest_leave),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('C',
+                                  "host-enter",
+                                  gettext_noop ("create a place"),
+                                  &op_host_enter),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('C',
+                                  "host-enter",
+                                  gettext_noop ("create a place"),
+                                  &op_host_enter),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('D',
+                                  "host-leave",
+                                  gettext_noop ("destroy a place we were hosting"),
+                                  &op_host_leave),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('E',
+                                  "guest-enter",
+                                  gettext_noop ("enter somebody else's place"),
+                                  &op_guest_enter),
+
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('F',
+                                  "look-for",
+                                  gettext_noop ("find state matching name prefix"),
+                                  &op_look_for),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('H',
+                                  "replay-latest",
+                                  gettext_noop ("replay history of messages up to the given --limit"),
+                                  &op_replay_latest),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('N',
+                                  "host-reconnect",
+                                  gettext_noop ("reconnect to a previously created place"),
+                                  &op_host_reconnect),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('P',
+                                  "host-announce",
+                                  gettext_noop ("publish something to a place we are hosting"),
+                                  &op_host_announce),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('R',
+                                  "guest-reconnect",
+                                  gettext_noop ("reconnect to a previously entered place"),
+                                  &op_guest_reconnect),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('S',
+                                  "look-at",
+                                  gettext_noop ("search for state matching exact name"),
+                                  &op_look_at),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('T',
+                                  "guest-talk",
+                                  gettext_noop ("submit something to somebody's place"),
+                                  &op_guest_talk),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('U',
+                                  "status",
+                                  gettext_noop ("list of egos and subscribed places"),
+                                  &op_status),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('X',
+                                  "replay",
+                                  gettext_noop ("extract and replay history between message IDs --start and --until"),
+                                  &op_replay),
 
 
     /* options */
 
-    { 'a', "app", "APPLICATION_ID",
-      gettext_noop ("application ID to use when connecting"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_app },
-
-    { 'd', "data", "DATA",
-      gettext_noop ("message body or state value"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_data },
-
-    { 'e', "ego", "NAME|PUBKEY",
-      gettext_noop ("name or public key of ego"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_ego },
-
-    { 'f', "follow", NULL,
-      gettext_noop ("wait for incoming messages"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &opt_follow },
-
-    { 'g', "gns", "GNS_NAME",
-      gettext_noop ("GNS name"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_gns },
-
-    { 'i', "peer", "PEER_ID",
-      gettext_noop ("peer ID for --guest-enter"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_peer },
-
-    { 'k', "name", "VAR_NAME",
-      gettext_noop ("name (key) to query from state"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_name },
-
-    { 'm', "method", "METHOD_NAME",
-      gettext_noop ("method name"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_method },
-
-    { 'n', "limit", NULL,
-      gettext_noop ("number of messages to replay from history"),
-      GNUNET_YES, &GNUNET_GETOPT_set_ulong, &opt_limit },
-
-    { 'p', "place", "PUBKEY",
-      gettext_noop ("key address of place"),
-      GNUNET_YES, &GNUNET_GETOPT_set_string, &opt_place },
-
-    { 's', "start", NULL,
-      gettext_noop ("start message ID for history replay"),
-      GNUNET_YES, &GNUNET_GETOPT_set_ulong, &opt_start },
-
-    { 'w', "welcome", NULL,
-      gettext_noop ("respond to entry requests by admitting all guests"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &opt_welcome },
-
-    { 'u', "until", NULL,
-      gettext_noop ("end message ID for history replay"),
-      GNUNET_YES, &GNUNET_GETOPT_set_ulong, &opt_until },
-
-    { 'y', "deny", NULL,
-      gettext_noop ("respond to entry requests by refusing all guests"),
-      GNUNET_NO, &GNUNET_GETOPT_set_one, &opt_deny },
+    GNUNET_GETOPT_OPTION_STRING ('a',
+                                 "app",
+                                 "APPLICATION_ID",
+                                 gettext_noop ("application ID to use when connecting"),
+                                 &opt_app),
+
+    GNUNET_GETOPT_OPTION_STRING ('d',
+                                 "data",
+                                 "DATA",
+                                 gettext_noop ("message body or state value"),
+                                 &opt_data),
+
+    GNUNET_GETOPT_OPTION_STRING ('e',
+                                 "ego",
+                                 "NAME|PUBKEY",
+                                 gettext_noop ("name or public key of ego"),
+                                 &opt_ego),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('f',
+                                  "follow",
+                                  gettext_noop ("wait for incoming messages"),
+                                  &opt_follow),
+
+    GNUNET_GETOPT_OPTION_STRING ('g',
+                                 "gns",
+                                 "GNS_NAME",
+                                 gettext_noop ("GNS name"),
+                                 &opt_gns),
+
+    GNUNET_GETOPT_OPTION_STRING ('i',
+                                 "peer",
+                                 "PEER_ID",
+                                 gettext_noop ("peer ID for --guest-enter"),
+                                 &opt_peer),
+
+    GNUNET_GETOPT_OPTION_STRING ('k',
+                                 "name",
+                                 "VAR_NAME",
+                                 gettext_noop ("name (key) to query from state"),
+                                 &opt_name),
+
+    GNUNET_GETOPT_OPTION_STRING ('m',
+                                 "method",
+                                 "METHOD_NAME",
+                                 gettext_noop ("method name"),
+                                 &opt_method),
+
+    GNUNET_GETOPT_OPTION_SET_ULONG ('n',
+                                    "limit",
+                                    NULL,
+                                    gettext_noop ("number of messages to replay from history"),
+                                    &opt_limit),
+
+    GNUNET_GETOPT_OPTION_STRING ('p',
+                                 "place",
+                                 "PUBKEY",
+                                 gettext_noop ("key address of place"),
+                                 &opt_place),
+
+    GNUNET_GETOPT_OPTION_SET_ULONG ('s',
+                                    "start",
+                                    NULL,
+                                    gettext_noop ("start message ID for history replay"),
+                                    &opt_start),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('w',
+                                  "welcome",
+                                  gettext_noop ("respond to entry requests by admitting all guests"),
+                                  &opt_welcome),
+
+    GNUNET_GETOPT_OPTION_SET_ULONG ('u',
+                                    "until",
+                                    NULL,
+                                    gettext_noop ("end message ID for history replay"),
+                                    &opt_until),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('y',
+                                  "deny",
+                                  gettext_noop ("respond to entry requests by refusing all guests"),
+                                  &opt_deny),
 
     GNUNET_GETOPT_OPTION_END
   };
@@ -1350,8 +1399,8 @@ main (int argc, char *const *argv)
     "gnunet-social --guest-leave --place <PUBKEY>\n"
     "gnunet-social --guest-talk --place <PUBKEY> --method <METHOD_NAME> --data <MESSAGE_BODY>\n"
     "\n"
-    "gnunet-social --history-replay --place <PUBKEY> --start <MSGID> --until <MSGID>  [--method <METHOD_PREFIX>]\n"
-    "gnunet-social --history-replay-latest --place <PUBKEY> --limit <MSG_LIMIT> [--method <METHOD_PREFIX>]\n"
+    "gnunet-social --replay --place <PUBKEY> --start <MSGID> --until <MSGID>  [--method <METHOD_PREFIX>]\n"
+    "gnunet-social --replay-latest --place <PUBKEY> --limit <MSG_LIMIT> [--method <METHOD_PREFIX>]\n"
     "\n"
     "gnunet-social --look-at --place <PUBKEY> --name <FULL_NAME>\n"
     "gnunet-social --look-for --place <PUBKEY> --name <NAME_PREFIX>\n";
diff --git a/src/social/social_api.c b/src/social/social_api.c
index a7fe0916f..af1d6e57e 100644
--- a/src/social/social_api.c
+++ b/src/social/social_api.c
@@ -2110,7 +2110,7 @@ GNUNET_SOCIAL_place_msg_proc_set (struct GNUNET_SOCIAL_Place *plc,
   GNUNET_assert (NULL != method_prefix);
   struct MsgProcRequest *mpreq;
   uint16_t method_size = strnlen (method_prefix,
-                                  GNUNET_SERVER_MAX_MESSAGE_SIZE
+                                  GNUNET_MAX_MESSAGE_SIZE
                                   - sizeof (*mpreq)) + 1;
   GNUNET_assert ('\0' == method_prefix[method_size - 1]);
 
@@ -2159,7 +2159,7 @@ place_history_replay (struct GNUNET_SOCIAL_Place *plc,
 
   GNUNET_assert (NULL != method_prefix);
   uint16_t method_size = strnlen (method_prefix,
-                                  GNUNET_SERVER_MAX_MESSAGE_SIZE
+                                  GNUNET_MAX_MESSAGE_SIZE
                                   - sizeof (*req)) + 1;
   GNUNET_assert ('\0' == method_prefix[method_size - 1]);
 
@@ -2285,7 +2285,7 @@ place_state_get (struct GNUNET_SOCIAL_Place *plc,
   look->op_id = GNUNET_OP_add (plc->op, &op_recv_state_result, look, NULL);
 
   GNUNET_assert (NULL != name);
-  size_t name_size = strnlen (name, GNUNET_SERVER_MAX_MESSAGE_SIZE
+  size_t name_size = strnlen (name, GNUNET_MAX_MESSAGE_SIZE
                               - sizeof (*req)) + 1;
   struct GNUNET_MQ_Envelope *
     env = GNUNET_MQ_msg_extra (req, name_size, type);
@@ -2426,7 +2426,7 @@ GNUNET_SOCIAL_zone_add_place (const struct GNUNET_SOCIAL_App *app,
   size_t relay_size = relay_count * sizeof (*relays);
   size_t payload_size = name_size + password_size + relay_size;
 
-  if (GNUNET_SERVER_MAX_MESSAGE_SIZE < sizeof (*preq) + payload_size)
+  if (GNUNET_MAX_MESSAGE_SIZE < sizeof (*preq) + payload_size)
     return GNUNET_SYSERR;
 
   struct GNUNET_MQ_Envelope *
@@ -2506,7 +2506,7 @@ GNUNET_SOCIAL_zone_add_nym (const struct GNUNET_SOCIAL_App *app,
   struct ZoneAddNymRequest *nreq;
 
   size_t name_size = strlen (name) + 1;
-  if (GNUNET_SERVER_MAX_MESSAGE_SIZE < sizeof (*nreq) + name_size)
+  if (GNUNET_MAX_MESSAGE_SIZE < sizeof (*nreq) + name_size)
     return GNUNET_SYSERR;
 
   struct GNUNET_MQ_Envelope *
diff --git a/src/sq/Makefile.am b/src/sq/Makefile.am
index c5f80bcf6..cfccd89f6 100644
--- a/src/sq/Makefile.am
+++ b/src/sq/Makefile.am
@@ -9,7 +9,7 @@ if USE_COVERAGE
   AM_CFLAGS = --coverage
 endif
 
-if HAVE_POSTGRESQL
+if HAVE_SQLITE
 lib_LTLIBRARIES = libgnunetsq.la
 endif
 
diff --git a/src/sq/sq.c b/src/sq/sq.c
index dc4416761..089ebf0ff 100644
--- a/src/sq/sq.c
+++ b/src/sq/sq.c
@@ -49,7 +49,14 @@ GNUNET_SQ_bind (sqlite3_stmt *stmt,
                        "sq",
                        _("Failure to bind %u-th SQL parameter\n"),
                        i);
-      return GNUNET_SYSERR;
+      if (SQLITE_OK !=
+          sqlite3_reset (stmt))
+      {
+        GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                         "sq",
+                         _("Failure in sqlite3_reset (!)\n"));
+        return GNUNET_SYSERR;
+      }
     }
     GNUNET_assert (0 != params[i].num_params);
     j += params[i].num_params;
@@ -83,7 +90,12 @@ GNUNET_SQ_extract_result (sqlite3_stmt *result,
                     j,
                     rs[i].result_size,
                     rs[i].dst))
+    {
+      for (unsigned int k=0;k<i;k++)
+        if (NULL != rs[k].cleaner)
+          rs[k].cleaner (rs[k].cls);
       return GNUNET_SYSERR;
+    }
     GNUNET_assert (0 != rs[i].num_params);
     j += rs[i].num_params;
   }
@@ -105,4 +117,24 @@ GNUNET_SQ_cleanup_result (struct GNUNET_SQ_ResultSpec *rs)
       rs[i].cleaner (rs[i].cls);
 }
 
+
+/**
+ * Reset @a stmt and log error.
+ *
+ * @param dbh database handle
+ * @param stmt statement to reset
+ */
+void
+GNUNET_SQ_reset (sqlite3 *dbh,
+                 sqlite3_stmt *stmt)
+{
+  if (SQLITE_OK !=
+      sqlite3_reset (stmt))
+    GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
+                     "sqlite",
+                     _("Failed to reset sqlite statement with error: %s\n"),
+                     sqlite3_errmsg (dbh));
+}
+
+
 /* end of sq.c */
diff --git a/src/sq/sq_query_helper.c b/src/sq/sq_query_helper.c
index 5529c5e6c..94a3a3f1c 100644
--- a/src/sq/sq_query_helper.c
+++ b/src/sq/sq_query_helper.c
@@ -90,6 +90,14 @@ bind_string (void *cls,
              sqlite3_stmt *stmt,
              unsigned int off)
 {
+  if (NULL == data)
+  {
+    if (SQLITE_OK !=
+        sqlite3_bind_null (stmt,
+                           (int) off))
+      return GNUNET_SYSERR;
+    return GNUNET_OK;
+  }
   if (SQLITE_OK !=
       sqlite3_bind_text (stmt,
                          (int) off,
@@ -235,6 +243,40 @@ GNUNET_SQ_query_param_rsa_signature (const struct GNUNET_CRYPTO_RsaSignature *x)
 
 
 /**
+ * Function called to convert input argument into SQL parameters.
+ *
+ * @param cls closure
+ * @param data pointer to input argument
+ * @param data_len number of bytes in @a data (if applicable)
+ * @param stmt sqlite statement to bind parameters for
+ * @param off offset of the argument to bind in @a stmt, numbered from 1,
+ *            so immediately suitable for passing to `sqlite3_bind`-functions.
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK on success
+ */
+static int
+bind_abstime (void *cls,
+              const void *data,
+              size_t data_len,
+              sqlite3_stmt *stmt,
+              unsigned int off)
+{
+  const struct GNUNET_TIME_Absolute *u = data;
+  struct GNUNET_TIME_Absolute abs;
+
+  abs = *u;
+  if (abs.abs_value_us > INT64_MAX)
+    abs.abs_value_us = INT64_MAX;
+  GNUNET_assert (sizeof (uint64_t) == data_len);
+  if (SQLITE_OK !=
+      sqlite3_bind_int64 (stmt,
+                          (int) off,
+                          (sqlite3_int64) abs.abs_value_us))
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
  * Generate query parameter for an absolute time value.
  * The database must store a 64-bit integer.
  *
@@ -243,7 +285,13 @@ GNUNET_SQ_query_param_rsa_signature (const struct GNUNET_CRYPTO_RsaSignature *x)
 struct GNUNET_SQ_QueryParam
 GNUNET_SQ_query_param_absolute_time (const struct GNUNET_TIME_Absolute *x)
 {
-  return GNUNET_SQ_query_param_uint64 (&x->abs_value_us);
+ struct GNUNET_SQ_QueryParam qp = {
+    .conv = &bind_abstime,
+    .data = x,
+    .size = sizeof (struct GNUNET_TIME_Absolute),
+    .num_params = 1
+  };
+  return qp;
 }
 
 
@@ -269,6 +317,8 @@ bind_nbotime (void *cls,
   struct GNUNET_TIME_Absolute abs;
 
   abs = GNUNET_TIME_absolute_ntoh (*u);
+  if (abs.abs_value_us > INT64_MAX)
+    abs.abs_value_us = INT64_MAX;
   GNUNET_assert (sizeof (uint64_t) == data_len);
   if (SQLITE_OK !=
       sqlite3_bind_int64 (stmt,
diff --git a/src/sq/sq_result_helper.c b/src/sq/sq_result_helper.c
index eaf606aa4..9579863b2 100644
--- a/src/sq/sq_result_helper.c
+++ b/src/sq/sq_result_helper.c
@@ -46,6 +46,15 @@ extract_var_blob (void *cls,
   const void *ret;
   void **rdst = (void **) dst;
 
+  if (SQLITE_NULL ==
+      sqlite3_column_type (result,
+                           column))
+  {
+    *rdst = NULL;
+    *dst_size = 0;
+    return GNUNET_YES;
+  }
+
   if (SQLITE_BLOB !=
       sqlite3_column_type (result,
                            column))
@@ -142,6 +151,14 @@ extract_fixed_blob (void *cls,
   int have;
   const void *ret;
 
+  if ( (0 == *dst_size) &&
+       (SQLITE_NULL ==
+        sqlite3_column_type (result,
+                             column)) )
+  {
+    return GNUNET_YES;
+  }
+
   if (SQLITE_BLOB !=
       sqlite3_column_type (result,
                            column))
@@ -211,6 +228,13 @@ extract_utf8_string (void *cls,
   const char *text;
   char **rdst = dst;
 
+  if (SQLITE_NULL ==
+      sqlite3_column_type (result,
+                           column))
+  {
+    *rdst = NULL;
+    return GNUNET_OK;
+  }
   if (SQLITE_TEXT !=
       sqlite3_column_type (result,
                            column))
@@ -459,6 +483,45 @@ GNUNET_SQ_result_spec_rsa_signature (struct GNUNET_CRYPTO_RsaSignature **sig)
 
 
 /**
+ * Extract absolute time value from a Postgres database @a result at row @a row.
+ *
+ * @param cls closure
+ * @param result where to extract data from
+ * @param column column to extract data from
+ * @param[in,out] dst_size where to store size of result, may be NULL
+ * @param[out] dst where to store the result
+ * @return
+ *   #GNUNET_YES if all results could be extracted
+ *   #GNUNET_SYSERR if a result was invalid (non-existing field or NULL)
+ */
+static int
+extract_abs_time (void *cls,
+                  sqlite3_stmt *result,
+                  unsigned int column,
+                  size_t *dst_size,
+                  void *dst)
+{
+  struct GNUNET_TIME_Absolute *u = dst;
+  struct GNUNET_TIME_Absolute t;
+
+  GNUNET_assert (sizeof (uint64_t) == *dst_size);
+  if (SQLITE_INTEGER !=
+      sqlite3_column_type (result,
+                           column))
+  {
+    GNUNET_break (0);
+    return GNUNET_SYSERR;
+  }
+  t.abs_value_us = (uint64_t) sqlite3_column_int64 (result,
+                                                    column);
+  if (INT64_MAX == t.abs_value_us)
+    t = GNUNET_TIME_UNIT_FOREVER_ABS;
+  *u = t;
+  return GNUNET_OK;
+}
+
+
+/**
  * Absolute time expected.
  *
  * @param[out] at where to store the result
@@ -467,7 +530,14 @@ GNUNET_SQ_result_spec_rsa_signature (struct GNUNET_CRYPTO_RsaSignature **sig)
 struct GNUNET_SQ_ResultSpec
 GNUNET_SQ_result_spec_absolute_time (struct GNUNET_TIME_Absolute *at)
 {
-  return GNUNET_SQ_result_spec_uint64 (&at->abs_value_us);
+  struct GNUNET_SQ_ResultSpec rs = {
+    .conv = &extract_abs_time,
+    .dst = at,
+    .dst_size = sizeof (struct GNUNET_TIME_Absolute),
+    .num_params = 1
+  };
+
+  return rs;
 }
 
 
@@ -503,6 +573,8 @@ extract_abs_time_nbo (void *cls,
   }
   t.abs_value_us = (uint64_t) sqlite3_column_int64 (result,
                                                     column);
+  if (INT64_MAX == t.abs_value_us)
+    t = GNUNET_TIME_UNIT_FOREVER_ABS;
   *u = GNUNET_TIME_absolute_hton (t);
   return GNUNET_OK;
 }
diff --git a/src/sq/test_sq.c b/src/sq/test_sq.c
index e6896861e..713809030 100644
--- a/src/sq/test_sq.c
+++ b/src/sq/test_sq.c
@@ -253,8 +253,12 @@ main(int argc,
   {
     fprintf (stderr,
              "Failed to create table\n");
-    sqlite3_close (dbh);
-    unlink ("test.db");
+    GNUNET_break (SQLITE_OK ==
+                  sqlite3_close (dbh));
+    if (0 != unlink ("test.db"))
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                                "unlink",
+                                "test.db");
     return 1;
   }
 
@@ -266,12 +270,14 @@ main(int argc,
   {
     fprintf (stderr,
              "Failed to drop table\n");
-    sqlite3_close (dbh);
-    unlink ("test.db");
-    return 1;
+    ret = 1;
   }
-  sqlite3_close (dbh);
-  unlink ("test.db");
+  GNUNET_break (SQLITE_OK ==
+                sqlite3_close (dbh));
+  if (0 != unlink ("test.db"))
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                              "unlink",
+                              "test.db");
   return ret;
 }
 
diff --git a/src/statistics/gnunet-service-statistics.c b/src/statistics/gnunet-service-statistics.c
index 161327421..0cb136b99 100644
--- a/src/statistics/gnunet-service-statistics.c
+++ b/src/statistics/gnunet-service-statistics.c
@@ -359,7 +359,7 @@ transmit (struct ClientEntry *ce,
 
   size = strlen (e->subsystem->service) + 1 +
     strlen (e->name) + 1;
-  GNUNET_assert (size < GNUNET_SERVER_MAX_MESSAGE_SIZE);
+  GNUNET_assert (size < GNUNET_MAX_MESSAGE_SIZE);
   env = GNUNET_MQ_msg_extra (m,
                              size,
                              GNUNET_MESSAGE_TYPE_STATISTICS_VALUE);
@@ -776,7 +776,7 @@ check_watch (void *cls,
   size_t size;
   const char *service;
   const char *name;
-  
+
   size = ntohs (message->size) - sizeof (struct GNUNET_MessageHeader);
   if (size !=
       GNUNET_STRINGS_buffer_tokenize ((const char *) &message[1],
@@ -870,7 +870,7 @@ handle_watch (void *cls,
 
 /**
  * Handle DISCONNECT-message.  Sync to disk and send
- * back a #GNUNET_MESSAGE_TYPE_STATISTICS_DISCONNECT_CONFIRM 
+ * back a #GNUNET_MESSAGE_TYPE_STATISTICS_DISCONNECT_CONFIRM
  * message.
  *
  * @param cls the `struct ClientEntry *`
@@ -984,6 +984,7 @@ client_disconnect_cb (void *cls,
       }
     }
   }
+  GNUNET_free (ce);
   if ( (0 == client_count) &&
        (GNUNET_YES == in_shutdown) )
     do_shutdown ();
@@ -992,7 +993,7 @@ client_disconnect_cb (void *cls,
 
 /**
  * We've read a `struct GNUNET_STATISTICS_SetMessage *` from
- * disk. Check that it is well-formed, and if so pass it to 
+ * disk. Check that it is well-formed, and if so pass it to
  * the handler for set messages.
  *
  * @param cls NULL
diff --git a/src/statistics/gnunet-statistics.c b/src/statistics/gnunet-statistics.c
index ed0c3f27d..6cfc56171 100644
--- a/src/statistics/gnunet-statistics.c
+++ b/src/statistics/gnunet-statistics.c
@@ -378,28 +378,44 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'n', "name", "NAME",
-     gettext_noop ("limit output to statistics for the given NAME"), 1,
-     &GNUNET_GETOPT_set_string, &name},
-    {'p', "persistent", NULL,
-     gettext_noop ("make the value being set persistent"), 0,
-     &GNUNET_GETOPT_set_one, &persistent},
-    {'s', "subsystem", "SUBSYSTEM",
-     gettext_noop ("limit output to the given SUBSYSTEM"), 1,
-     &GNUNET_GETOPT_set_string, &subsystem},
-    {'q', "quiet", NULL,
-     gettext_noop ("just print the statistics value"), 0,
-     &GNUNET_GETOPT_set_one, &quiet},
-    {'w', "watch", NULL,
-     gettext_noop ("watch value continuously"), 0,
-     &GNUNET_GETOPT_set_one, &watch},
-     {'r', "remote", NULL,
-      gettext_noop ("connect to remote host"), 1,
-      &GNUNET_GETOPT_set_string, &remote_host},
-    {'o', "port", NULL,
-     gettext_noop ("port for remote host"), 1,
-     &GNUNET_GETOPT_set_uint, &remote_port},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_STRING ('n',
+                                 "name",
+                                 "NAME",
+                                 gettext_noop ("limit output to statistics for the given NAME"),
+                                 &name),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('p',
+                                  "persistent",
+                                  gettext_noop ("make the value being set persistent"),
+                                  &persistent),
+
+    GNUNET_GETOPT_OPTION_STRING ('s',
+                                 "subsystem",
+                                 "SUBSYSTEM",
+                                 gettext_noop ("limit output to the given SUBSYSTEM"),
+                                 &subsystem),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('q',
+                                  "quiet",
+                                  gettext_noop ("just print the statistics value"),
+                                  &quiet),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('w',
+                                  "watch",
+                                  gettext_noop ("watch value continuously"),
+                                  &watch),
+
+    GNUNET_GETOPT_OPTION_STRING ('r',
+                                 "remote",
+                                 "REMOTE",
+                                 gettext_noop ("connect to remote host"),
+                                 &remote_host),
+    GNUNET_GETOPT_OPTION_SET_ULONG ('o',
+                                    "port",
+                                    "PORT",
+                                    gettext_noop ("port for remote host"),
+                                    &remote_port),
     GNUNET_GETOPT_OPTION_END
   };
   remote_port = 0;
diff --git a/src/statistics/statistics_api.c b/src/statistics/statistics_api.c
index ad4453b2a..9d04e854f 100644
--- a/src/statistics/statistics_api.c
+++ b/src/statistics/statistics_api.c
@@ -349,7 +349,7 @@ schedule_watch_request (struct GNUNET_STATISTICS_Handle *h,
   slen = strlen (watch->subsystem) + 1;
   nlen = strlen (watch->name) + 1;
   nsize = sizeof (struct GNUNET_MessageHeader) + slen + nlen;
-  if (nsize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (nsize >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -1098,7 +1098,7 @@ GNUNET_STATISTICS_get (struct GNUNET_STATISTICS_Handle *handle,
   slen1 = strlen (subsystem) + 1;
   slen2 = strlen (name) + 1;
   GNUNET_assert (slen1 + slen2 + sizeof (struct GNUNET_MessageHeader) <
-                 GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                 GNUNET_MAX_MESSAGE_SIZE);
   ai = GNUNET_new (struct GNUNET_STATISTICS_GetHandle);
   ai->sh = handle;
   ai->subsystem = GNUNET_strdup (subsystem);
@@ -1246,7 +1246,7 @@ add_setter_action (struct GNUNET_STATISTICS_Handle *h,
   slen = strlen (h->subsystem) + 1;
   nlen = strlen (name) + 1;
   nsize = sizeof (struct GNUNET_STATISTICS_SetMessage) + slen + nlen;
-  if (nsize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (nsize >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
diff --git a/src/testbed-logger/gnunet-service-testbed-logger.c b/src/testbed-logger/gnunet-service-testbed-logger.c
index 1c250b306..f915e70af 100644
--- a/src/testbed-logger/gnunet-service-testbed-logger.c
+++ b/src/testbed-logger/gnunet-service-testbed-logger.c
@@ -40,42 +40,6 @@
   LOG (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__)
 
 /**
- * The message queue for sending messages to clients
- */
-struct MessageQueue
-{
-  /**
-   * The message to be sent
-   */
-  struct GNUNET_MessageHeader *msg;
-
-  /**
-   * The client to send the message to
-   */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * next pointer for DLL
-   */
-  struct MessageQueue *next;
-
-  /**
-   * prev pointer for DLL
-   */
-  struct MessageQueue *prev;
-};
-
-/**
- * The message queue head
- */
-static struct MessageQueue *mq_head;
-
-/**
- * The message queue tail
- */
-static struct MessageQueue *mq_tail;
-
-/**
  * Handle for buffered writing.
  */
 struct GNUNET_BIO_WriteHandle *bio;
@@ -92,23 +56,38 @@ static int in_shutdown;
 
 
 /**
- * Message handler for #GNUNET_MESSAGE_TYPE_TESTBED_ADDHOST messages
+ * Check #GNUNET_MESSAGE_TYPE_TESTBED_LOGGER_MSG messages
  *
- * @param cls NULL
- * @param client identification of the client
+ * @param cls client identification of the client
+ * @param msg the actual message
+ * @return #GNUNET_OK (they are all always OK)
+ */
+static int
+check_log_msg (void *cls,
+               const struct GNUNET_MessageHeader *msg)
+{
+  return GNUNET_OK;
+}
+
+
+/**
+ * Message handler for #GNUNET_MESSAGE_TYPE_TESTBED_LOGGER_MSG messages
+ *
+ * @param cls client identification of the client
  * @param msg the actual message
  */
 static void
 handle_log_msg (void *cls,
-                struct GNUNET_SERVER_Client *client,
                 const struct GNUNET_MessageHeader *msg)
 {
+  struct GNUNET_SERVICE_Client *client = cls;
   uint16_t ms;
 
-  ms = ntohs (msg->size);
-  ms -= sizeof (struct GNUNET_MessageHeader);
-  GNUNET_BIO_write (bio, &msg[1], ms);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  ms = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader);
+  GNUNET_BIO_write (bio,
+                    &msg[1],
+                    ms);
+  GNUNET_SERVICE_client_continue (client);
 }
 
 
@@ -120,69 +99,55 @@ handle_log_msg (void *cls,
 static void
 shutdown_task (void *cls)
 {
-  struct MessageQueue *mq_entry;
-
   in_shutdown = GNUNET_YES;
   if (0 != nconn)
   {
     /* Delay shutdown if there are active connections */
-    GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
+    GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                   NULL);
     return;
   }
-  while (NULL != (mq_entry = mq_head))
-  {
-    GNUNET_free (mq_entry->msg);
-    GNUNET_SERVER_client_drop (mq_entry->client);
-    GNUNET_CONTAINER_DLL_remove (mq_head,
-                                 mq_tail,
-                                 mq_entry);
-    GNUNET_free (mq_entry);
-  }
-  GNUNET_break (GNUNET_OK == GNUNET_BIO_write_close (bio));
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_BIO_write_close (bio));
 }
 
 
 /**
-x * Functions with this signature are called whenever a client
- * is disconnected on the network level.
+ * Callback called when a client connects to the service.
  *
- * @param cls closure
- * @param client identification of the client; NULL
- *        for the last call when the server is destroyed
+ * @param cls closure for the service
+ * @param c the new client that connected to the service
+ * @param mq the message queue used to send messages to the client
+ * @return @a c
  */
-static void
-client_disconnected (void *cls,
-                     struct GNUNET_SERVER_Client *client)
+static void *
+client_connect_cb (void *cls,
+                   struct GNUNET_SERVICE_Client *c,
+                   struct GNUNET_MQ_Handle *mq)
 {
-  if (NULL == client)
-  {
-    GNUNET_break (0 == nconn);
-    return;
-  }
-  nconn--;
-  if (GNUNET_YES == in_shutdown)
-    GNUNET_SCHEDULER_shutdown ();
+  /* FIXME: is this really what we want here? */
+  GNUNET_SERVICE_client_persist (c);
+  nconn++;
+  return c;
 }
 
 
 /**
- * Functions with this signature are called whenever a client
- * is connected on the network level.
+ * Callback called when a client disconnected from the service
  *
- * @param cls closure
- * @param client identification of the client
+ * @param cls closure for the service
+ * @param c the client that disconnected
+ * @param internal_cls should be equal to @a c
  */
 static void
-client_connected (void *cls,
-                  struct GNUNET_SERVER_Client *client)
+client_disconnect_cb (void *cls,
+                      struct GNUNET_SERVICE_Client *c,
+                      void *internal_cls)
 {
-  if (NULL == client)
-  {
-    GNUNET_break (0 == nconn);
-    return;
-  }
-  GNUNET_SERVER_client_persist_ (client);
-  nconn++;
+  nconn--;
+  if (GNUNET_YES == in_shutdown)
+    GNUNET_SCHEDULER_shutdown ();
+  GNUNET_assert (c == internal_cls);
 }
 
 
@@ -190,18 +155,14 @@ client_connected (void *cls,
  * Testbed setup
  *
  * @param cls closure
- * @param server the initialized server
  * @param cfg configuration to use
+ * @param service the initialized service
  */
 static void
 logger_run (void *cls,
-            struct GNUNET_SERVER_Handle *server,
-            const struct GNUNET_CONFIGURATION_Handle *cfg)
+            const struct GNUNET_CONFIGURATION_Handle *cfg,
+            struct GNUNET_SERVICE_Handle *service)
 {
-  static const struct GNUNET_SERVER_MessageHandler message_handlers[] = {
-    {&handle_log_msg, NULL, GNUNET_MESSAGE_TYPE_TESTBED_LOGGER_MSG, 0},
-    {NULL, NULL, 0, 0}
-  };
   char *dir;
   char *fn;
   char *hname;
@@ -223,7 +184,8 @@ logger_run (void *cls,
   pid = getpid ();
   hname_len = GNUNET_OS_get_hostname_max_length ();
   hname = GNUNET_malloc (hname_len);
-  if (0 != gethostname (hname, hname_len))
+  if (0 != gethostname (hname,
+                        hname_len))
   {
     LOG (GNUNET_ERROR_TYPE_ERROR,
          "Cannot get hostname.  Exiting\n");
@@ -247,24 +209,27 @@ logger_run (void *cls,
     return;
   }
   GNUNET_free (fn);
-  GNUNET_SERVER_add_handlers (server, message_handlers);
-  GNUNET_SERVER_connect_notify (server, &client_connected, NULL);
-  GNUNET_SERVER_disconnect_notify (server, &client_disconnected, NULL);
-  GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
+  GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                 NULL);
   LOG_DEBUG ("TESTBED-LOGGER startup complete\n");
 }
 
 
 /**
- * The starting point of execution
+ * Define "main" method using service macro.
  */
-int
-main (int argc, char *const *argv)
-{
-  return (GNUNET_OK ==
-          GNUNET_SERVICE_run (argc, argv, "testbed-logger",
-                              GNUNET_SERVICE_OPTION_NONE,
-                              &logger_run, NULL)) ? 0 : 1;
-}
+GNUNET_SERVICE_MAIN
+("testbed-logger",
+ GNUNET_SERVICE_OPTION_NONE,
+ &logger_run,
+ &client_connect_cb,
+ &client_disconnect_cb,
+ NULL,
+ GNUNET_MQ_hd_var_size (log_msg,
+                        GNUNET_MESSAGE_TYPE_TESTBED_LOGGER_MSG,
+                        struct GNUNET_MessageHeader,
+                        NULL),
+ GNUNET_MQ_handler_end ());
+
 
 /* end of gnunet-service-testbed-logger.c */
diff --git a/src/testbed-logger/test_testbed_logger_api.c b/src/testbed-logger/test_testbed_logger_api.c
index 0ebe0c3f4..e627feeb4 100644
--- a/src/testbed-logger/test_testbed_logger_api.c
+++ b/src/testbed-logger/test_testbed_logger_api.c
@@ -258,11 +258,15 @@ main (int argc, char **argv)
   GNUNET_log_setup ("test-testbed-logger-api",
                     "WARNING",
                     NULL);
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_DISK_directory_remove ("/tmp/test-testbed"));
   ret = GNUNET_TESTING_service_run ("test-testbed-logger",
                                     "testbed-logger",
                                     "test_testbed_logger_api.conf",
                                     &test_main,
                                     NULL);
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_DISK_directory_remove ("/tmp/test-testbed"));
   if (0 != ret)
     return 1;
   if (GNUNET_OK != result)
diff --git a/src/testbed-logger/testbed_logger_api.c b/src/testbed-logger/testbed_logger_api.c
index 25494aed0..7aa4ade35 100644
--- a/src/testbed-logger/testbed_logger_api.c
+++ b/src/testbed-logger/testbed_logger_api.c
@@ -39,7 +39,7 @@
 /**
  * The size of the buffer we fill before sending out the message
  */
-#define BUFFER_SIZE (GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct GNUNET_MessageHeader))
+#define BUFFER_SIZE (GNUNET_MAX_MESSAGE_SIZE - sizeof (struct GNUNET_MessageHeader))
 
 /**
  * Connection handle for the logger service
diff --git a/src/testbed/Makefile.am b/src/testbed/Makefile.am
index 94c908295..61cfba2ff 100644
--- a/src/testbed/Makefile.am
+++ b/src/testbed/Makefile.am
@@ -104,13 +104,13 @@ libgnunettestbed_la_SOURCES = \
   testbed_api_test.c \
   testbed_api_topology.c testbed_api_topology.h \
   testbed_api_sd.c testbed_api_sd.h \
-  testbed_api_barriers.c 
+  testbed_api_barriers.c
 libgnunettestbed_la_LIBADD = $(XLIB) \
  $(top_builddir)/src/core/libgnunetcore.la \
  $(top_builddir)/src/statistics/libgnunetstatistics.la \
  $(top_builddir)/src/transport/libgnunettransport.la \
  $(top_builddir)/src/hello/libgnunethello.la \
- -lm \
+ -lm $(Z_LIBS) \
  $(top_builddir)/src/util/libgnunetutil.la \
  $(top_builddir)/src/testing/libgnunettesting.la \
  $(LTLIBINTL)
diff --git a/src/testbed/generate-underlay-topology.c b/src/testbed/generate-underlay-topology.c
index 36580a2a3..ab7d81c8b 100644
--- a/src/testbed/generate-underlay-topology.c
+++ b/src/testbed/generate-underlay-topology.c
@@ -70,7 +70,7 @@ enum GNUNET_TESTBED_TopologyOption topology;
 /**
  * The number of peers to include in the topology
  */
-static int num_peers;
+static unsigned int num_peers;
 
 /**
  * program result
@@ -335,11 +335,15 @@ int
 main (int argc, char *const argv[])
 {
   struct GNUNET_GETOPT_CommandLineOption option[] = {
-    {'p', "num-peers", "COUNT",
-     gettext_noop ("create COUNT number of peers"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_peers},
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('p',
+                                   "num-peers",
+                                   "COUNT",
+                                   gettext_noop ("create COUNT number of peers"),
+                                   &num_peers),
     GNUNET_GETOPT_OPTION_END
   };
+
   int ret;
 
   exit_result = GNUNET_SYSERR;
diff --git a/src/testbed/gnunet-helper-testbed.c b/src/testbed/gnunet-helper-testbed.c
index 6368fb74b..d2a3a98b7 100644
--- a/src/testbed/gnunet-helper-testbed.c
+++ b/src/testbed/gnunet-helper-testbed.c
@@ -495,7 +495,7 @@ error:
 static void
 read_task (void *cls)
 {
-  char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE];
+  char buf[GNUNET_MAX_MESSAGE_SIZE];
   ssize_t sread;
 
   read_task_id = NULL;
diff --git a/src/testbed/gnunet-service-testbed_barriers.c b/src/testbed/gnunet-service-testbed_barriers.c
index a201d22bb..681ed6df2 100644
--- a/src/testbed/gnunet-service-testbed_barriers.c
+++ b/src/testbed/gnunet-service-testbed_barriers.c
@@ -478,7 +478,7 @@ GST_barriers_init (struct GNUNET_CONFIGURATION_Handle *cfg)
   LOG_DEBUG ("Launching testbed-barrier service\n");
   barrier_map = GNUNET_CONTAINER_multihashmap_create (3,
                                                       GNUNET_YES);
-  ctx = GNUNET_SERVICE_starT ("testbed-barrier",
+  ctx = GNUNET_SERVICE_start ("testbed-barrier",
                               cfg,
                               &connect_cb,
                               &disconnect_cb,
@@ -524,7 +524,7 @@ GST_barriers_destroy ()
                                                         NULL));
   GNUNET_CONTAINER_multihashmap_destroy (barrier_map);
   GNUNET_assert (NULL != ctx);
-  GNUNET_SERVICE_stoP (ctx);
+  GNUNET_SERVICE_stop (ctx);
 }
 
 
diff --git a/src/testbed/gnunet-service-testbed_oc.c b/src/testbed/gnunet-service-testbed_oc.c
index b775f31bd..09849797c 100644
--- a/src/testbed/gnunet-service-testbed_oc.c
+++ b/src/testbed/gnunet-service-testbed_oc.c
@@ -371,7 +371,7 @@ GST_cleanup_focc (struct ForwardedOverlayConnectContext *focc)
 /**
  * Timeout task for cancelling a forwarded overlay connect connect
  *
- * @param cls the ForwardedOverlayConnectContext
+ * @param cls the `struct ForwardedOperationContext`
  */
 static void
 forwarded_overlay_connect_timeout (void *cls)
diff --git a/src/testbed/gnunet-service-testbed_peers.c b/src/testbed/gnunet-service-testbed_peers.c
index a977b2b9b..aacd62583 100644
--- a/src/testbed/gnunet-service-testbed_peers.c
+++ b/src/testbed/gnunet-service-testbed_peers.c
@@ -1137,14 +1137,8 @@ arm_req_string (enum GNUNET_ARM_RequestStatus rs)
   {
   case GNUNET_ARM_REQUEST_SENT_OK:
     return _("Message was sent successfully");
-  case GNUNET_ARM_REQUEST_CONFIGURATION_ERROR:
-    return _("Misconfiguration (can't connect to the ARM service)");
   case GNUNET_ARM_REQUEST_DISCONNECTED:
     return _("We disconnected from ARM before we could send a request");
-  case GNUNET_ARM_REQUEST_BUSY:
-    return _("ARM API is busy");
-  case GNUNET_ARM_REQUEST_TIMEOUT:
-    return _("Request timed out");
   }
   return _("Unknown request status");
 }
diff --git a/src/testbed/gnunet-testbed-profiler.c b/src/testbed/gnunet-testbed-profiler.c
index 9468b3c91..29f77193d 100644
--- a/src/testbed/gnunet-testbed-profiler.c
+++ b/src/testbed/gnunet-testbed-profiler.c
@@ -276,23 +276,34 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'p', "num-peers", "COUNT",
-     gettext_noop ("create COUNT number of peers"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_peers},
-    {'e', "num-errors", "COUNT",
-     gettext_noop ("tolerate COUNT number of continious timeout failures"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_cont_fails},
-    {'n', "non-interactive", NULL,
-     gettext_noop ("run profiler in non-interactive mode where upon "
-                   "testbed setup the profiler does not wait for a "
-                   "keystroke but continues to run until a termination "
-                   "signal is received"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &noninteractive},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('p',
+                                   "num-peers",
+                                   "COUNT",
+                                   gettext_noop ("create COUNT number of peers"),
+                                   &num_peers),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('e',
+                                   "num-errors",
+                                   "COUNT",
+                                   gettext_noop ("tolerate COUNT number of continious timeout failures"),
+                                   &num_cont_fails),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "non-interactive",
+                                  gettext_noop ("run profiler in non-interactive mode where upon "
+                                                "testbed setup the profiler does not wait for a "
+                                                "keystroke but continues to run until a termination "
+                                                "signal is received"),
+                                  &noninteractive),
+
 #if !ENABLE_SUPERMUC
-    {'H', "hosts", "FILENAME",
-     gettext_noop ("name of the file with the login information for the testbed"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &hosts_file},
+    GNUNET_GETOPT_OPTION_STRING ('H',
+                                 "hosts",
+                                 "FILENAME",
+                                 gettext_noop ("name of the file with the login information for the testbed"),
+                                 &hosts_file),
 #endif
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/testbed/test_gnunet_helper_testbed.c b/src/testbed/test_gnunet_helper_testbed.c
index 7a2dce8a5..7ba9e7c31 100644
--- a/src/testbed/test_gnunet_helper_testbed.c
+++ b/src/testbed/test_gnunet_helper_testbed.c
@@ -121,10 +121,12 @@ do_abort (void *cls)
  *               #GNUNET_SYSERR during GNUNET_HELPER_stop()
  */
 static void
-cont_cb (void *cls, int result)
+cont_cb (void *cls,
+         int result)
 {
   shandle = NULL;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Message sent\n");
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Message sent\n");
   GNUNET_assert (GNUNET_OK == result);
 }
 
@@ -138,11 +140,11 @@ cont_cb (void *cls, int result)
  * @param cls closure
  * @param client identification of the client
  * @param message the actual message
- *
  * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
  */
 static int
-mst_cb (void *cls, void *client, const struct GNUNET_MessageHeader *message)
+mst_cb (void *cls,
+        const struct GNUNET_MessageHeader *message)
 {
   const struct GNUNET_TESTBED_HelperReply *msg;
   char *config;
@@ -207,8 +209,12 @@ run (void *cls, char *const *args, const char *cfgfile,
   const char *trusted_ip = "127.0.0.1";
 
   helper =
-      GNUNET_HELPER_start (GNUNET_YES, "gnunet-helper-testbed", binary_argv,
-                           &mst_cb, &exp_cb, NULL);
+      GNUNET_HELPER_start (GNUNET_YES,
+                           "gnunet-helper-testbed",
+                           binary_argv,
+                           &mst_cb,
+                           &exp_cb,
+                           NULL);
   GNUNET_assert (NULL != helper);
   cfg = GNUNET_CONFIGURATION_dup (cfg2);
   msg = GNUNET_TESTBED_create_helper_init_msg_ (trusted_ip, NULL, cfg);
diff --git a/src/testbed/testbed_api_hosts.c b/src/testbed/testbed_api_hosts.c
index 731944bc4..5d2c1cc37 100644
--- a/src/testbed/testbed_api_hosts.c
+++ b/src/testbed/testbed_api_hosts.c
@@ -952,10 +952,11 @@ gen_rsh_suffix_args (const char * const *append_args)
  * @param client identification of the client
  * @param message the actual message
  *
- * @return GNUNET_OK on success, GNUNET_SYSERR to stop further processing
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
  */
 static int
-helper_mst (void *cls, void *client, const struct GNUNET_MessageHeader *message)
+helper_mst (void *cls,
+            const struct GNUNET_MessageHeader *message)
 {
   struct GNUNET_TESTBED_ControllerProc *cp = cls;
   const struct GNUNET_TESTBED_HelperReply *msg;
diff --git a/src/testbed/testbed_api_peers.c b/src/testbed/testbed_api_peers.c
index 2af62b44a..871e554a9 100644
--- a/src/testbed/testbed_api_peers.c
+++ b/src/testbed/testbed_api_peers.c
@@ -969,7 +969,7 @@ GNUNET_TESTBED_peer_manage_service (void *op_cls,
   GNUNET_assert (TESTBED_PS_STARTED == peer->state); /* peer is not running? */
   msize = strlen (service_name) + 1;
   msize += sizeof (struct GNUNET_TESTBED_ManagePeerServiceMessage);
-  if (GNUNET_SERVER_MAX_MESSAGE_SIZE < msize)
+  if (GNUNET_MAX_MESSAGE_SIZE < msize)
     return NULL;
   data = GNUNET_new (struct ManageServiceData);
   data->cb = cb;
diff --git a/src/testbed/testbed_api_topology.c b/src/testbed/testbed_api_topology.c
index a21a7cf53..7bc36d1b4 100644
--- a/src/testbed/testbed_api_topology.c
+++ b/src/testbed/testbed_api_topology.c
@@ -1147,9 +1147,11 @@ gen_topo_from_file (struct TopologyContext *tc,
              other_peer_id);
       while (('\n' != data[offset]) && ('|' != data[offset]) && (offset < fs))
         offset++;
-      if ('\n' == data[offset])
+      if ( (offset < fs) &&
+           ('\n' == data[offset]) )
         state = PEER_INDEX;
-      else if ('|' == data[offset])
+      else if ( (offset < fs) &&
+                ('|' == data[offset]) )
       {
         state = OTHER_PEER_INDEX;
         offset++;
@@ -1491,7 +1493,7 @@ GNUNET_TESTBED_topology_get_ (enum GNUNET_TESTBED_TopologyOption *topology,
     {
       if (NULL != topology)
         *topology = (enum GNUNET_TESTBED_TopologyOption) cnt;
-      GNUNET_assert (GNUNET_TESTBED_TOPOLOGY_OPTION_END != *topology);
+      GNUNET_assert (GNUNET_TESTBED_TOPOLOGY_OPTION_END != (enum GNUNET_TESTBED_TopologyOption) cnt);
       return GNUNET_YES;
     }
   }
diff --git a/src/testing/gnunet-testing.c b/src/testing/gnunet-testing.c
index 07f1560cb..686b38192 100644
--- a/src/testing/gnunet-testing.c
+++ b/src/testing/gnunet-testing.c
@@ -348,17 +348,35 @@ run_no_scheduler (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'C', "cfg", NULL, gettext_noop ("create unique configuration files"),
-     GNUNET_NO, &GNUNET_GETOPT_set_one, &create_cfg},
-    {'k', "key", "FILENAME", gettext_noop ("extract hostkey file from pre-computed hostkey list"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &create_hostkey},
-    {'n', "number", "NUMBER", gettext_noop ("number of unique configuration files to create, or number of the hostkey to extract"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &create_no},
-    {'t', "template", "FILENAME", gettext_noop ("configuration template"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &create_cfg_template},
-    {'r', "run", "SERVICE", gettext_noop ("run the given service, wait on stdin for 'r' (restart) or 'q' (quit)"),
-     GNUNET_YES, &GNUNET_GETOPT_set_string, &run_service_name},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('C',
+                                  "cfg",
+                                  gettext_noop ("create unique configuration files"),
+                                  &create_cfg),
+    GNUNET_GETOPT_OPTION_STRING ('k',
+                                 "key",
+                                 "FILENAME",
+                                 gettext_noop ("extract hostkey file from pre-computed hostkey list"),
+                                 &create_hostkey),
+
+    GNUNET_GETOPT_OPTION_SET_UINT ('n',
+                                   "number",
+                                   "NUMBER",
+                                   gettext_noop ("number of unique configuration files to create, or number of the hostkey to extract"),
+                                   &create_no),
+
+
+    GNUNET_GETOPT_OPTION_STRING ('t',
+                                 "template",
+                                 "FILENAME",
+                                 gettext_noop ("configuration template"),
+                                 &create_cfg_template),
+
+    GNUNET_GETOPT_OPTION_STRING ('r',
+                                 "run",
+                                 "SERVICE",
+                                 gettext_noop ("run the given service, wait on stdin for 'r' (restart) or 'q' (quit)"),
+                                 &run_service_name),
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
diff --git a/src/testing/list-keys.c b/src/testing/list-keys.c
index 1fd46ebf2..cfb49460a 100644
--- a/src/testing/list-keys.c
+++ b/src/testing/list-keys.c
@@ -6,10 +6,6 @@ static unsigned int nkeys;
 static unsigned int nskip;
 static int result;
 
-
-
-
-
 /**
  * Main run function.
  *
@@ -87,12 +83,11 @@ run (void *cls, char *const *args, const char *cfgfile,
 int main (int argc, char *argv[])
 {
   struct GNUNET_GETOPT_CommandLineOption option[] = {
-    {'n', "num-keys", "COUNT",
-     gettext_noop ("list COUNT number of keys"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &nkeys},
-    {'s', "skip", "COUNT",
-     gettext_noop ("skip COUNT number of keys in the beginning"),
-     GNUNET_YES, &GNUNET_GETOPT_set_uint, &nskip},
+    GNUNET_GETOPT_OPTION_SET_UINT ('n',
+                                   "num-keys",
+                                   "COUNT",
+                                   gettext_noop ("list COUNT number of keys"),
+                                   &nkeys),
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/testing/testing.c b/src/testing/testing.c
index bab7976ea..ba7bf827e 100644
--- a/src/testing/testing.c
+++ b/src/testing/testing.c
@@ -907,6 +907,7 @@ update_config_sections (void *cls,
       {
         ikeys[key] = ptr;
         ptr = strstr (ptr, ";");
+        GNUNET_assert (NULL != ptr); /* worked just before... */
         *ptr = '\0';
         ptr++;
       }
diff --git a/src/topology/friends.c b/src/topology/friends.c
index 59c70e4b0..a960fad17 100644
--- a/src/topology/friends.c
+++ b/src/topology/friends.c
@@ -58,14 +58,20 @@ GNUNET_FRIENDS_parse (const struct GNUNET_CONFIGURATION_Handle *cfg,
     return GNUNET_SYSERR;
   }
   if ( (GNUNET_OK != GNUNET_DISK_file_test (fn)) &&
-       (GNUNET_OK != GNUNET_DISK_fn_write (fn, NULL, 0,
+       (GNUNET_OK != GNUNET_DISK_fn_write (fn,
+                                           NULL,
+                                           0,
                                            GNUNET_DISK_PERM_USER_READ |
-                                           GNUNET_DISK_PERM_USER_WRITE)) )
-      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "write", fn);
+                                           GNUNET_DISK_PERM_USER_WRITE |
+                                           GNUNET_DISK_OPEN_CREATE)) )
+    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
+                              "write",
+                              fn);
   if ( (GNUNET_OK !=
         GNUNET_DISK_file_size (fn,
                                &fsize,
-                               GNUNET_NO, GNUNET_YES)) ||
+                               GNUNET_NO,
+                               GNUNET_YES)) ||
        (0 == fsize) )
   {
     GNUNET_free (fn);
@@ -93,8 +99,8 @@ GNUNET_FRIENDS_parse (const struct GNUNET_CONFIGURATION_Handle *cfg,
       pos++;
     if (GNUNET_OK !=
         GNUNET_CRYPTO_eddsa_public_key_from_string (&data[start],
-                                                       pos - start,
-                                                       &pid.public_key))
+                                                    pos - start,
+                                                    &pid.public_key))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                   _("Syntax error in FRIENDS file at offset %llu, skipping bytes `%.*s'.\n"),
diff --git a/src/topology/gnunet-daemon-topology.c b/src/topology/gnunet-daemon-topology.c
index 067ebce23..537ffe059 100644
--- a/src/topology/gnunet-daemon-topology.c
+++ b/src/topology/gnunet-daemon-topology.c
@@ -499,7 +499,7 @@ schedule_next_hello (void *cls)
   /* find applicable HELLOs */
   fah.peer = pl;
   fah.result = NULL;
-  fah.max_size = GNUNET_SERVER_MAX_MESSAGE_SIZE - 1;
+  fah.max_size = GNUNET_MAX_MESSAGE_SIZE - 1;
   fah.next_adv = GNUNET_TIME_UNIT_FOREVER_REL;
   GNUNET_CONTAINER_multipeermap_iterate (peers,
                                          &find_advertisable_hello,
diff --git a/src/transport/Makefile.am b/src/transport/Makefile.am
index acc2557c6..7687f2348 100644
--- a/src/transport/Makefile.am
+++ b/src/transport/Makefile.am
@@ -191,6 +191,8 @@ libexec_PROGRAMS = \
  $(BT_BIN) \
  gnunet-service-transport
 
+
+
 bin_PROGRAMS = \
  gnunet-transport \
  gnunet-transport-certificate-creation
@@ -561,7 +563,7 @@ TESTS = \
  $(HTTP_API_TIMEOUT_TEST) \
  $(HTTPS_API_TIMEOUT_TEST) \
  $(WLAN_TIMEOUT_TEST) \
- $(BT_TIMEOUT_TEST) 
+ $(BT_TIMEOUT_TEST)
 if HAVE_GETOPT_BINARY
 TESTS += \
 test_transport_api_slow_ats
diff --git a/src/transport/gnunet-helper-transport-wlan-dummy.c b/src/transport/gnunet-helper-transport-wlan-dummy.c
index 684546314..63ed9c4b7 100644
--- a/src/transport/gnunet-helper-transport-wlan-dummy.c
+++ b/src/transport/gnunet-helper-transport-wlan-dummy.c
@@ -120,11 +120,11 @@ send_mac_to_plugin (char *buffer, struct GNUNET_TRANSPORT_WLAN_MacAddress *mac)
  * type to the output forward and copy it to the buffer for stdout.
  *
  * @param cls the 'struct SendBuffer' to copy the converted message to
- * @param client unused
  * @param hdr inbound message from the FIFO
  */
 static int
-stdin_send (void *cls, void *client, const struct GNUNET_MessageHeader *hdr)
+stdin_send (void *cls,
+            const struct GNUNET_MessageHeader *hdr)
 {
   struct SendBuffer *write_pout = cls;
   const struct GNUNET_TRANSPORT_WLAN_RadiotapSendMessage *in;
@@ -166,11 +166,11 @@ stdin_send (void *cls, void *client, const struct GNUNET_MessageHeader *hdr)
  * We read a full message from stdin.  Copy it to our send buffer.
  *
  * @param cls the 'struct SendBuffer' to copy to
- * @param client unused
  * @param hdr the message we received to copy to the buffer
  */
 static int
-file_in_send (void *cls, void *client, const struct GNUNET_MessageHeader *hdr)
+file_in_send (void *cls,
+              const struct GNUNET_MessageHeader *hdr)
 {
   struct SendBuffer *write_std = cls;
   uint16_t sendsize;
@@ -213,8 +213,8 @@ main (int argc, char *argv[])
   fd_set wfds;
   struct timeval tv;
   int retval;
-  struct GNUNET_SERVER_MessageStreamTokenizer *stdin_mst = NULL;
-  struct GNUNET_SERVER_MessageStreamTokenizer *file_in_mst = NULL;
+  struct GNUNET_MessageStreamTokenizer *stdin_mst = NULL;
+  struct GNUNET_MessageStreamTokenizer *file_in_mst = NULL;
   struct GNUNET_TRANSPORT_WLAN_MacAddress macaddr;
   int first;
 
@@ -340,8 +340,8 @@ main (int argc, char *argv[])
   write_std.pos = 0;
   write_pout.size = 0;
   write_pout.pos = 0;
-  stdin_mst = GNUNET_SERVER_mst_create (&stdin_send, &write_pout);
-  file_in_mst = GNUNET_SERVER_mst_create (&file_in_send, &write_std);
+  stdin_mst = GNUNET_MST_create (&stdin_send, &write_pout);
+  file_in_mst = GNUNET_MST_create (&file_in_send, &write_std);
 
   /* Send 'random' mac address */
   macaddr.mac[0] = 0x13;
@@ -453,8 +453,9 @@ main (int argc, char *argv[])
       }
       else if (0 < readsize)
       {
-        GNUNET_SERVER_mst_receive (stdin_mst, NULL, readbuf, readsize,
-                                   GNUNET_NO, GNUNET_NO);
+        GNUNET_MST_from_buffer (stdin_mst,
+                                readbuf, readsize,
+                                GNUNET_NO, GNUNET_NO);
 
       }
       else
@@ -475,8 +476,9 @@ main (int argc, char *argv[])
       }
       else if (0 < readsize)
       {
-        GNUNET_SERVER_mst_receive (file_in_mst, NULL, readbuf, readsize,
-                                   GNUNET_NO, GNUNET_NO);
+        GNUNET_MST_from_buffer (file_in_mst,
+                                readbuf, readsize,
+                                GNUNET_NO, GNUNET_NO);
       }
       else
       {
@@ -489,9 +491,9 @@ main (int argc, char *argv[])
 end:
   /* clean up */
   if (NULL != stdin_mst)
-    GNUNET_SERVER_mst_destroy (stdin_mst);
+    GNUNET_MST_destroy (stdin_mst);
   if (NULL != file_in_mst)
-    GNUNET_SERVER_mst_destroy (file_in_mst);
+    GNUNET_MST_destroy (file_in_mst);
 
   if (NULL != fpout)
     fclose (fpout);
diff --git a/src/transport/gnunet-service-transport_validation.c b/src/transport/gnunet-service-transport_validation.c
index 505626b59..4a6d427be 100644
--- a/src/transport/gnunet-service-transport_validation.c
+++ b/src/transport/gnunet-service-transport_validation.c
@@ -599,7 +599,7 @@ transmit_ping_if_allowed (void *cls,
   ping.challenge = htonl (ve->challenge);
   ping.target = *pid;
 
-  if (tsize >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (tsize >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     hsize = 0;
diff --git a/src/transport/gnunet-transport-certificate-creation.c b/src/transport/gnunet-transport-certificate-creation.c
index 129df7b35..edaf4773a 100644
--- a/src/transport/gnunet-transport-certificate-creation.c
+++ b/src/transport/gnunet-transport-certificate-creation.c
@@ -44,7 +44,7 @@ make_dev_zero (int fd,
   GNUNET_assert (-1 != z);
   if (z == fd)
     return;
-  dup2 (z, fd);
+  GNUNET_break (fd == dup2 (z, fd));
   GNUNET_assert (0 == close (z));
 }
 #endif
diff --git a/src/transport/gnunet-transport-profiler.c b/src/transport/gnunet-transport-profiler.c
index dceff7e3b..9292c42d4 100644
--- a/src/transport/gnunet-transport-profiler.c
+++ b/src/transport/gnunet-transport-profiler.c
@@ -151,7 +151,7 @@ static struct GNUNET_PeerIdentity pid;
 /**
  * Selected level of verbosity.
  */
-static int verbosity;
+static unsigned int verbosity;
 
 
 /**
@@ -520,7 +520,7 @@ run (void *cls,
   cfg = (struct GNUNET_CONFIGURATION_Handle *) mycfg;
 
   ret = 1;
-  if (GNUNET_SERVER_MAX_MESSAGE_SIZE <= benchmark_size)
+  if (GNUNET_MAX_MESSAGE_SIZE <= benchmark_size)
   {
     FPRINTF (stderr,
              "Message size too big!\n");
@@ -610,26 +610,36 @@ main (int argc, char * const *argv)
   benchmark_iterations = DEFAULT_ITERATION_COUNT;
   benchmark_running = GNUNET_NO;
 
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-
-    { 's', "send", NULL,
-      gettext_noop ("send data to peer"),
-      0, &GNUNET_GETOPT_set_one, &benchmark_send},
-    { 'r', "receive", NULL, gettext_noop
-      ("receive data from peer"), 0,
-      &GNUNET_GETOPT_set_one, &benchmark_receive},
-    { 'i', "iterations", NULL, gettext_noop
-      ("iterations"), 1,
-      &GNUNET_GETOPT_set_uint, &benchmark_iterations},
-    { 'n', "number", NULL, gettext_noop
-      ("number of messages to send"), 1,
-      &GNUNET_GETOPT_set_uint, &benchmark_count},
-    { 'm', "messagesize", NULL, gettext_noop
-      ("message size to use"), 1,
-      &GNUNET_GETOPT_set_uint, &benchmark_size},
-    { 'p', "peer", "PEER",
-      gettext_noop ("peer identity"), 1, &GNUNET_GETOPT_set_string,
-      &cpid },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('s',
+                                  "send",
+                                  gettext_noop ("send data to peer"),
+                                  &benchmark_send),
+    GNUNET_GETOPT_OPTION_SET_ONE ('r',
+                                  "receive",
+                                  gettext_noop ("receive data from peer"),
+                                  &benchmark_receive),
+    GNUNET_GETOPT_OPTION_SET_UINT ('i',
+                                   "iterations",
+                                   NULL,
+                                   gettext_noop ("iterations"),
+                                   &benchmark_iterations),
+    GNUNET_GETOPT_OPTION_SET_UINT ('n',
+                                   "number",
+                                   NULL,
+                                   gettext_noop ("number of messages to send"),
+                                   &benchmark_count),
+    GNUNET_GETOPT_OPTION_SET_UINT ('m',
+                                   "messagesize",
+                                   NULL,
+                                   gettext_noop ("message size to use"),
+                                   &benchmark_size),
+    GNUNET_GETOPT_OPTION_STRING ('p',
+                                 "peer",
+                                 "PEER",
+                                 gettext_noop ("peer identity"),
+                                 &cpid),
     GNUNET_GETOPT_OPTION_VERBOSE (&verbosity),
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/transport/gnunet-transport.c b/src/transport/gnunet-transport.c
index fcfc94ac8..8624b09b4 100644
--- a/src/transport/gnunet-transport.c
+++ b/src/transport/gnunet-transport.c
@@ -167,11 +167,6 @@ struct PeerResolutionContext
 #define BLOCKSIZE 4
 
 /**
- * Which peer should we connect to?
- */
-static char *cpid;
-
-/**
  * Handle to transport service.
  */
 static struct GNUNET_TRANSPORT_CoreHandle *handle;
@@ -283,7 +278,7 @@ static struct GNUNET_TRANSPORT_PluginMonitor *pm;
 
 /**
  * Identity of the peer we transmit to / connect to.
- * (equivalent to 'cpid' string).
+ * ('-p' command-line option).
  */
 static struct GNUNET_PeerIdentity pid;
 
@@ -295,7 +290,7 @@ static struct GNUNET_SCHEDULER_Task *op_timeout;
 /**
  * Selected level of verbosity.
  */
-static int verbosity;
+static unsigned int verbosity;
 
 /**
  * Resolver process handle.
@@ -485,7 +480,7 @@ operation_timeout (void *cls)
                _("Failed to resolve address for peer `%s'\n"),
                GNUNET_i2s (&cur->addrcp->peer));
 
-      GNUNET_CONTAINER_DLL_remove(rc_head, 
+      GNUNET_CONTAINER_DLL_remove(rc_head,
                                   rc_tail,
                                   cur);
       GNUNET_TRANSPORT_address_to_string_cancel (cur->asc);
@@ -729,7 +724,7 @@ print_info (const struct GNUNET_PeerIdentity *id,
              GNUNET_STRINGS_absolute_time_to_string (state_timeout));
   }
   else if ( (GNUNET_YES == iterate_connections) &&
-            (GNUNET_TRANSPORT_is_connected(state)) ) 
+            (GNUNET_TRANSPORT_is_connected(state)) )
   {
     /* Only connected peers, skip state */
     FPRINTF (stdout,
@@ -1089,10 +1084,9 @@ plugin_monitoring_cb (void *cls,
     }
     return; /* shutdown */
   }
-  if ( (NULL != cpid) &&
-       (0 != memcmp (&info->address->peer,
-                     cpid,
-                     sizeof (struct GNUNET_PeerIdentity))) )
+  if (0 != memcmp (&info->address->peer,
+                   &pid,
+                   sizeof (struct GNUNET_PeerIdentity)))
     return; /* filtered */
   if (NULL == addr)
   {
@@ -1241,21 +1235,11 @@ run (void *cls,
      const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *mycfg)
 {
+  static struct GNUNET_PeerIdentity zero_pid;
   int counter = 0;
   ret = 1;
 
   cfg = (struct GNUNET_CONFIGURATION_Handle *) mycfg;
-  if ( (NULL != cpid) &&
-       (GNUNET_OK !=
-        GNUNET_CRYPTO_eddsa_public_key_from_string (cpid,
-                                                    strlen (cpid),
-                                                    &pid.public_key)))
-  {
-    FPRINTF (stderr,
-             _("Failed to parse peer identity `%s'\n"),
-             cpid);
-    return;
-  }
 
   counter = benchmark_send + benchmark_receive + iterate_connections
       + monitor_connections + monitor_connects + do_disconnect +
@@ -1290,7 +1274,9 @@ run (void *cls,
 
   if (do_disconnect) /* -D: Disconnect from peer */
   {
-    if (NULL == cpid)
+    if (0 == memcmp (&zero_pid,
+                     &pid,
+                     sizeof (pid)))
     {
       FPRINTF (stderr,
                _("Option `%s' makes no sense without option `%s'.\n"),
@@ -1315,7 +1301,9 @@ run (void *cls,
   }
   else if (benchmark_send) /* -s: Benchmark sending */
   {
-    if (NULL == cpid)
+    if (0 == memcmp (&zero_pid,
+                     &pid,
+                     sizeof (pid)))
     {
       FPRINTF (stderr,
                _("Option `%s' makes no sense without option `%s'.\n"),
@@ -1352,7 +1340,7 @@ run (void *cls,
                              NULL),
       GNUNET_MQ_handler_end ()
     };
-    
+
     handle = GNUNET_TRANSPORT_core_connect (cfg,
                                             NULL,
                                             handlers,
@@ -1378,7 +1366,7 @@ run (void *cls,
   else if (iterate_connections) /* -i: List information about peers once */
   {
     pic = GNUNET_TRANSPORT_monitor_peers (cfg,
-                                          (NULL == cpid) ? NULL : &pid,
+                                          &pid,
                                           GNUNET_YES,
                                           &process_peer_iteration_cb,
                                           (void *) cfg);
@@ -1391,7 +1379,7 @@ run (void *cls,
     monitored_peers = GNUNET_CONTAINER_multipeermap_create (10,
                                                             GNUNET_NO);
     pic = GNUNET_TRANSPORT_monitor_peers (cfg,
-                                          (NULL == cpid) ? NULL : &pid,
+                                          &pid,
                                           GNUNET_NO,
                                           &process_peer_monitoring_cb,
                                           NULL);
@@ -1439,37 +1427,49 @@ main (int argc,
       char * const *argv)
 {
   int res;
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'a', "all", NULL,
-      gettext_noop ("print information for all peers (instead of only connected peers)"),
-      0, &GNUNET_GETOPT_set_one, &iterate_all },
-    { 'b', "benchmark", NULL,
-      gettext_noop ("measure how fast we are receiving data from all peers (until CTRL-C)"),
-      0, &GNUNET_GETOPT_set_one, &benchmark_receive },
-    { 'D', "disconnect",
-      NULL, gettext_noop ("disconnect from a peer"), 0,
-      &GNUNET_GETOPT_set_one, &do_disconnect },
-    { 'i', "information", NULL,
-      gettext_noop ("provide information about all current connections (once)"),
-      0, &GNUNET_GETOPT_set_one, &iterate_connections },
-    { 'm', "monitor", NULL,
-      gettext_noop ("provide information about all current connections (continuously)"),
-      0, &GNUNET_GETOPT_set_one, &monitor_connections },
-    { 'e', "events", NULL,
-      gettext_noop ("provide information about all connects and disconnect events (continuously)"),
-      0, &GNUNET_GETOPT_set_one, &monitor_connects },
-    { 'n', "numeric",
-      NULL, gettext_noop ("do not resolve hostnames"), 0,
-      &GNUNET_GETOPT_set_one, &numeric },
-    { 'p', "peer", "PEER",
-      gettext_noop ("peer identity"), 1, &GNUNET_GETOPT_set_string,
-      &cpid },
-    { 'P', "plugins", NULL,
-      gettext_noop ("monitor plugin sessions"), 0, &GNUNET_GETOPT_set_one,
-      &monitor_plugins },
-    { 's', "send", NULL, gettext_noop
-      ("send data for benchmarking to the other peer (until CTRL-C)"), 0,
-      &GNUNET_GETOPT_set_one, &benchmark_send },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('a',
+                                  "all",
+                                  gettext_noop ("print information for all peers (instead of only connected peers)"),
+                                  &iterate_all),
+    GNUNET_GETOPT_OPTION_SET_ONE ('b',
+                                  "benchmark",
+                                  gettext_noop ("measure how fast we are receiving data from all peers (until CTRL-C)"),
+                                  &benchmark_receive),
+    GNUNET_GETOPT_OPTION_SET_ONE ('D',
+                                  "disconnect",
+                                  gettext_noop ("disconnect from a peer"),
+                                  &do_disconnect),
+    GNUNET_GETOPT_OPTION_SET_ONE ('i',
+                                  "information",
+                                  gettext_noop ("provide information about all current connections (once)"),
+                                  &iterate_connections),
+    GNUNET_GETOPT_OPTION_SET_ONE ('m',
+                                  "monitor",
+                                  gettext_noop ("provide information about all current connections (continuously)"),
+                                  &monitor_connections),
+    GNUNET_GETOPT_OPTION_SET_ONE ('e',
+                                  "events",
+                                  gettext_noop ("provide information about all connects and disconnect events (continuously)"),
+                                  &monitor_connects),
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "numeric",
+                                  gettext_noop ("do not resolve hostnames"),
+                                  &numeric),
+    GNUNET_GETOPT_OPTION_SET_BASE32_AUTO ('p',
+                                          "peer",
+                                          "PEER",
+                                          gettext_noop ("peer identity"),
+                                          &pid),
+    GNUNET_GETOPT_OPTION_SET_ONE ('P',
+                                  "plugins",
+                                  gettext_noop ("monitor plugin sessions"),
+                                  &monitor_plugins),
+    GNUNET_GETOPT_OPTION_SET_ONE ('s',
+                                  "send",
+                                  gettext_noop
+      ("send data for benchmarking to the other peer (until CTRL-C)"),
+                                  &benchmark_send),
     GNUNET_GETOPT_OPTION_VERBOSE (&verbosity),
     GNUNET_GETOPT_OPTION_END
   };
diff --git a/src/transport/plugin_transport_http_client.c b/src/transport/plugin_transport_http_client.c
index ceed94af8..e20948c5a 100644
--- a/src/transport/plugin_transport_http_client.c
+++ b/src/transport/plugin_transport_http_client.c
@@ -221,7 +221,7 @@ struct GNUNET_ATS_Session
   /**
    * Message stream tokenizer for incoming data
    */
-  struct GNUNET_SERVER_MessageStreamTokenizer *msg_tk;
+  struct GNUNET_MessageStreamTokenizer *msg_tk;
 
   /**
    * Session timeout task
@@ -528,7 +528,7 @@ client_delete_session (struct GNUNET_ATS_Session *s)
                           GNUNET_TRANSPORT_SS_DONE);
   if (NULL != s->msg_tk)
   {
-    GNUNET_SERVER_mst_destroy (s->msg_tk);
+    GNUNET_MST_destroy (s->msg_tk);
     s->msg_tk = NULL;
   }
   GNUNET_HELLO_address_free (s->address);
@@ -1158,13 +1158,11 @@ client_wake_up (void *cls)
  * Callback for message stream tokenizer
  *
  * @param cls the session
- * @param client not used
  * @param message the message received
  * @return always #GNUNET_OK
  */
 static int
 client_receive_mst_cb (void *cls,
-                       void *client,
                        const struct GNUNET_MessageHeader *message)
 {
   struct GNUNET_ATS_Session *s = cls;
@@ -1274,14 +1272,13 @@ client_receive (void *stream,
     return CURL_WRITEFUNC_PAUSE;
   }
   if (NULL == s->msg_tk)
-    s->msg_tk = GNUNET_SERVER_mst_create (&client_receive_mst_cb,
-                                          s);
-  GNUNET_SERVER_mst_receive (s->msg_tk,
-                             s,
-                             stream,
-                             len,
-                             GNUNET_NO,
-                             GNUNET_NO);
+    s->msg_tk = GNUNET_MST_create (&client_receive_mst_cb,
+                                   s);
+  GNUNET_MST_from_buffer (s->msg_tk,
+                          stream,
+                          len,
+                          GNUNET_NO,
+                          GNUNET_NO);
   return len;
 }
 
@@ -1641,7 +1638,7 @@ client_connect_get (struct GNUNET_ATS_Session *s)
                     CURLOPT_CONNECTTIMEOUT_MS,
                     (long) (HTTP_CLIENT_NOT_VALIDATED_TIMEOUT.rel_value_us / 1000LL));
   curl_easy_setopt (s->get.easyhandle, CURLOPT_BUFFERSIZE,
-                    2 * GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                    2 * GNUNET_MAX_MESSAGE_SIZE);
 #if CURL_TCP_NODELAY
   curl_easy_setopt (ps->recv_endpoint,
                     CURLOPT_TCP_NODELAY,
@@ -1818,7 +1815,7 @@ client_connect_put (struct GNUNET_ATS_Session *s)
                     CURLOPT_CONNECTTIMEOUT_MS,
                     (long) (HTTP_CLIENT_NOT_VALIDATED_TIMEOUT.rel_value_us / 1000LL));
   curl_easy_setopt (s->put.easyhandle, CURLOPT_BUFFERSIZE,
-                    2 * GNUNET_SERVER_MAX_MESSAGE_SIZE);
+                    2 * GNUNET_MAX_MESSAGE_SIZE);
 #if CURL_TCP_NODELAY
   curl_easy_setopt (s->put.easyhandle, CURLOPT_TCP_NODELAY, 1);
 #endif
diff --git a/src/transport/plugin_transport_http_server.c b/src/transport/plugin_transport_http_server.c
index 63c67b81c..ff2d68602 100644
--- a/src/transport/plugin_transport_http_server.c
+++ b/src/transport/plugin_transport_http_server.c
@@ -201,7 +201,7 @@ struct GNUNET_ATS_Session
   /**
    * Message stream tokenizer for incoming data
    */
-  struct GNUNET_SERVER_MessageStreamTokenizer *msg_tk;
+  struct GNUNET_MessageStreamTokenizer *msg_tk;
 
   /**
    * Client recv handle
@@ -608,7 +608,7 @@ server_delete_session (struct GNUNET_ATS_Session *s)
   }
   if (NULL != s->msg_tk)
   {
-    GNUNET_SERVER_mst_destroy (s->msg_tk);
+    GNUNET_MST_destroy (s->msg_tk);
     s->msg_tk = NULL;
   }
   GNUNET_HELLO_address_free (s->address);
@@ -1621,13 +1621,11 @@ server_send_callback (void *cls,
  * Callback called by MessageStreamTokenizer when a message has arrived
  *
  * @param cls current session as closure
- * @param client client
  * @param message the message to be forwarded to transport service
  * @return #GNUNET_OK
  */
 static int
 server_receive_mst_cb (void *cls,
-                       void *client,
                        const struct GNUNET_MessageHeader *message)
 {
   struct GNUNET_ATS_Session *s = cls;
@@ -1847,13 +1845,16 @@ server_access_cb (void *cls,
              *upload_data_size);
         if (s->msg_tk == NULL)
         {
-          s->msg_tk = GNUNET_SERVER_mst_create (&server_receive_mst_cb, s);
+          s->msg_tk = GNUNET_MST_create (&server_receive_mst_cb,
+                                         s);
         }
-        GNUNET_SERVER_mst_receive (s->msg_tk, s, upload_data, *upload_data_size,
-            GNUNET_NO, GNUNET_NO);
+        GNUNET_MST_from_buffer (s->msg_tk,
+                                upload_data,
+                                *upload_data_size,
+                                GNUNET_NO, GNUNET_NO);
         server_mhd_connection_timeout (plugin, s,
-            GNUNET_CONSTANTS_IDLE_CONNECTION_TIMEOUT.rel_value_us / 1000LL
-                / 1000LL);
+                                       GNUNET_CONSTANTS_IDLE_CONNECTION_TIMEOUT.rel_value_us / 1000LL
+                                       / 1000LL);
         (*upload_data_size) = 0;
       }
       else
@@ -1935,7 +1936,7 @@ server_disconnect_cb (void *cls,
       sc->session->server_recv = NULL;
       if (NULL != sc->session->msg_tk)
       {
-        GNUNET_SERVER_mst_destroy (sc->session->msg_tk);
+        GNUNET_MST_destroy (sc->session->msg_tk);
         sc->session->msg_tk = NULL;
       }
     }
@@ -2223,7 +2224,7 @@ run_mhd_start_daemon (struct HTTP_Server_Plugin *plugin,
                              timeout,
                              MHD_OPTION_CONNECTION_MEMORY_LIMIT,
                              (size_t) (2 *
-                                       GNUNET_SERVER_MAX_MESSAGE_SIZE),
+                                       GNUNET_MAX_MESSAGE_SIZE),
                              MHD_OPTION_NOTIFY_COMPLETED,
                              &server_disconnect_cb, plugin,
                              MHD_OPTION_EXTERNAL_LOGGER,
@@ -2757,7 +2758,7 @@ server_start_report_addresses (struct HTTP_Server_Plugin *plugin)
     return;
   }
 
-  plugin->nat 
+  plugin->nat
     = GNUNET_NAT_register (plugin->env->cfg,
                            "transport-http_server",
                            IPPROTO_TCP,
diff --git a/src/transport/plugin_transport_tcp.c b/src/transport/plugin_transport_tcp.c
index 34bbd00e0..a63013caa 100644
--- a/src/transport/plugin_transport_tcp.c
+++ b/src/transport/plugin_transport_tcp.c
@@ -45,10 +45,538 @@
  */
 #define NAT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 10)
 
-GNUNET_NETWORK_STRUCT_BEGIN
+/**
+ * Opaque handle that can be used to cancel
+ * a transmit-ready notification.
+ */
+struct GNUNET_CONNECTION_TransmitHandle;
+
+/**
+ * @brief handle for a server
+ */
+struct GNUNET_SERVER_Handle;
+
+/**
+ * @brief opaque handle for a client of the server
+ */
+struct GNUNET_SERVER_Client;
+
+/**
+ * @brief opaque handle server returns for aborting transmission to a client.
+ */
+struct GNUNET_SERVER_TransmitHandle;
+
+/**
+ * @brief handle for a network connection
+ */
+struct GNUNET_CONNECTION_Handle;
+
+/**
+ * @brief handle for a network service
+ */
+struct LEGACY_SERVICE_Context;
+
+
+/**
+ * Stops a service that was started with #GNUNET_SERVICE_start().
+ *
+ * @param srv service to stop
+ */
+void
+LEGACY_SERVICE_stop (struct LEGACY_SERVICE_Context *srv);
+
+
+
+/**
+ * Function called to notify a client about the connection begin ready
+ * to queue more data.  @a buf will be NULL and @a size zero if the
+ * connection was closed for writing in the meantime.
+ *
+ * @param cls closure
+ * @param size number of bytes available in @a buf
+ * @param buf where the callee should write the message
+ * @return number of bytes written to @a buf
+ */
+typedef size_t
+(*GNUNET_CONNECTION_TransmitReadyNotify) (void *cls,
+                                          size_t size,
+                                          void *buf);
+
+/**
+ * Credentials for UNIX domain sockets.
+ */
+struct GNUNET_CONNECTION_Credentials
+{
+  /**
+   * UID of the other end of the connection.
+   */
+  uid_t uid;
+
+  /**
+   * GID of the other end of the connection.
+   */
+  gid_t gid;
+};
+
+
+/**
+ * Functions with this signature are called whenever a client
+ * is disconnected on the network level.
+ *
+ * @param cls closure
+ * @param client identification of the client; NULL
+ *        for the last call when the server is destroyed
+ */
+typedef void
+(*GNUNET_SERVER_DisconnectCallback) (void *cls,
+                                     struct GNUNET_SERVER_Client *client);
+
+
+/**
+ * Functions with this signature are called whenever a client
+ * is connected on the network level.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ */
+typedef void
+(*GNUNET_SERVER_ConnectCallback) (void *cls,
+                                  struct GNUNET_SERVER_Client *client);
+
+
+
+
+/**
+ * Function to call for access control checks.
+ *
+ * @param cls closure
+ * @param ucred credentials, if available, otherwise NULL
+ * @param addr address
+ * @param addrlen length of address
+ * @return GNUNET_YES to allow, GNUNET_NO to deny, GNUNET_SYSERR
+ *   for unknown address family (will be denied).
+ */
+typedef int
+(*GNUNET_CONNECTION_AccessCheck) (void *cls,
+                                  const struct
+                                  GNUNET_CONNECTION_Credentials *
+                                  ucred,
+                                  const struct sockaddr * addr,
+                                  socklen_t addrlen);
+
+/**
+ * Callback function for data received from the network.  Note that
+ * both "available" and "err" would be 0 if the read simply timed out.
+ *
+ * @param cls closure
+ * @param buf pointer to received data
+ * @param available number of bytes availabe in "buf",
+ *        possibly 0 (on errors)
+ * @param addr address of the sender
+ * @param addrlen size of addr
+ * @param errCode value of errno (on errors receiving)
+ */
+typedef void
+(*GNUNET_CONNECTION_Receiver) (void *cls, const void *buf,
+                               size_t available,
+                               const struct sockaddr * addr,
+                               socklen_t addrlen, int errCode);
+
+
+
+/**
+ * Close the connection and free associated resources.  There must
+ * not be any pending requests for reading or writing to the
+ * connection at this time.
+ *
+ * @param connection connection to destroy
+ */
+void
+GNUNET_CONNECTION_destroy (struct GNUNET_CONNECTION_Handle *connection);
+
+
+/**
+ * Signature of a function to create a custom tokenizer.
+ *
+ * @param cls closure from #GNUNET_SERVER_set_callbacks
+ * @param client handle to client the tokenzier will be used for
+ * @return handle to custom tokenizer ('mst')
+ */
+typedef void*
+(*GNUNET_SERVER_MstCreateCallback) (void *cls,
+                                    struct GNUNET_SERVER_Client *client);
+
+
+/**
+ * Signature of a function to destroy a custom tokenizer.
+ *
+ * @param cls closure from #GNUNET_SERVER_set_callbacks
+ * @param mst custom tokenizer handle
+ */
+typedef void
+(*GNUNET_SERVER_MstDestroyCallback) (void *cls,
+                                     void *mst);
+
+/**
+ * Signature of a function to receive data for a custom tokenizer.
+ *
+ * @param cls closure from #GNUNET_SERVER_set_callbacks
+ * @param mst custom tokenizer handle
+ * @param client_identity ID of client for which this is a buffer,
+ *        can be NULL (will be passed back to 'cb')
+ * @param buf input data to add
+ * @param size number of bytes in @a buf
+ * @param purge should any excess bytes in the buffer be discarded
+ *       (i.e. for packet-based services like UDP)
+ * @param one_shot only call callback once, keep rest of message in buffer
+ * @return #GNUNET_OK if we are done processing (need more data)
+ *         #GNUNET_NO if one_shot was set and we have another message ready
+ *         #GNUNET_SYSERR if the data stream is corrupt
+ */
+typedef int
+(*GNUNET_SERVER_MstReceiveCallback) (void *cls, void *mst,
+                                     struct GNUNET_SERVER_Client *client,
+                                     const char *buf,
+                                     size_t size,
+                                     int purge,
+                                     int one_shot);
+/**
+ * Functions with this signature are called whenever a message is
+ * received.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ * @param message the actual message
+ */
+typedef void
+(*GNUNET_SERVER_MessageCallback) (void *cls,
+                                  struct GNUNET_SERVER_Client *client,
+                                  const struct GNUNET_MessageHeader *message);
+
+/**
+ * Message handler.  Each struct specifies how to handle on particular
+ * type of message received.
+ */
+struct GNUNET_SERVER_MessageHandler
+{
+  /**
+   * Function to call for messages of "type".
+   */
+  GNUNET_SERVER_MessageCallback callback;
+
+  /**
+   * Closure argument for @e callback.
+   */
+  void *callback_cls;
+
+  /**
+   * Type of the message this handler covers.
+   */
+  uint16_t type;
+
+  /**
+   * Expected size of messages of this type.  Use 0 for
+   * variable-size.  If non-zero, messages of the given
+   * type will be discarded (and the connection closed)
+   * if they do not have the right size.
+   */
+  uint16_t expected_size;
+
+};
+
+
+/**
+ * Options for the service (bitmask).
+ */
+enum LEGACY_SERVICE_Options
+{
+  /**
+   * Use defaults.  Terminates all client connections and the listen
+   * sockets immediately upon receiving the shutdown signal.
+   */
+  LEGACY_SERVICE_OPTION_NONE = 0,
+
+  /**
+   * Do not trigger server shutdown on signal at all; instead, allow
+   * for the user to terminate the server explicitly when needed
+   * by calling #LEGACY_SERVICE_shutdown().
+   */
+  LEGACY_SERVICE_OPTION_MANUAL_SHUTDOWN = 1,
+
+  /**
+   * Trigger a SOFT server shutdown on signals, allowing active
+   * non-monitor clients to complete their transactions.
+   */
+  LEGACY_SERVICE_OPTION_SOFT_SHUTDOWN = 2
+};
+
+
+
+/**
+ * Ask the server to disconnect from the given client.  This is the
+ * same as passing #GNUNET_SYSERR to #GNUNET_SERVER_receive_done,
+ * except that it allows dropping of a client even when not handling a
+ * message from that client.
+ *
+ * @param client the client to disconnect from
+ */
+void
+GNUNET_SERVER_client_disconnect (struct GNUNET_SERVER_Client *client);
+
+/**
+ * Return user context associated with the given client.
+ * Note: you should probably use the macro (call without the underscore).
+ *
+ * @param client client to query
+ * @param size number of bytes in user context struct (for verification only)
+ * @return pointer to user context
+ */
+void *
+GNUNET_SERVER_client_get_user_context_ (struct GNUNET_SERVER_Client *client,
+                                        size_t size);
+
+
+/**
+ * Functions with this signature are called whenever a
+ * complete message is received by the tokenizer.
+ *
+ * Do not call #GNUNET_SERVER_mst_destroy from within
+ * the scope of this callback.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ * @param message the actual message
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
+ */
+typedef int
+(*GNUNET_SERVER_MessageTokenizerCallback) (void *cls,
+                                           void *client,
+                                           const struct GNUNET_MessageHeader *message);
+
+
+/**
+ * Create a message stream tokenizer.
+ *
+ * @param cb function to call on completed messages
+ * @param cb_cls closure for @a cb
+ * @return handle to tokenizer
+ */
+struct GNUNET_SERVER_MessageStreamTokenizer *
+GNUNET_SERVER_mst_create (GNUNET_SERVER_MessageTokenizerCallback cb,
+                          void *cb_cls);
+
+/**
+ * Add incoming data to the receive buffer and call the
+ * callback for all complete messages.
+ *
+ * @param mst tokenizer to use
+ * @param client_identity ID of client for which this is a buffer,
+ *        can be NULL (will be passed back to 'cb')
+ * @param buf input data to add
+ * @param size number of bytes in @a buf
+ * @param purge should any excess bytes in the buffer be discarded
+ *       (i.e. for packet-based services like UDP)
+ * @param one_shot only call callback once, keep rest of message in buffer
+ * @return #GNUNET_OK if we are done processing (need more data)
+ *         #GNUNET_NO if one_shot was set and we have another message ready
+ *         #GNUNET_SYSERR if the data stream is corrupt
+ */
+int
+GNUNET_SERVER_mst_receive (struct GNUNET_SERVER_MessageStreamTokenizer *mst,
+                           void *client_identity,
+                           const char *buf, size_t size,
+                           int purge, int one_shot);
+
+
+
+/**
+ * Destroys a tokenizer.
+ *
+ * @param mst tokenizer to destroy
+ */
+void
+GNUNET_SERVER_mst_destroy (struct GNUNET_SERVER_MessageStreamTokenizer *mst);
 
 
 /**
+ * Set user context to be associated with the given client.
+ * Note: you should probably use the macro (call without the underscore).
+ *
+ * @param client client to query
+ * @param ptr pointer to user context
+ * @param size number of bytes in user context struct (for verification only)
+ */
+void
+GNUNET_SERVER_client_set_user_context_ (struct GNUNET_SERVER_Client *client,
+                                        void *ptr,
+                                        size_t size);
+/**
+ * Return user context associated with the given client.
+ *
+ * @param client client to query
+ * @param type expected return type (i.e. 'struct Foo')
+ * @return pointer to user context of type 'type *'.
+ */
+#define GNUNET_SERVER_client_get_user_context(client,type)              \
+  (type *) GNUNET_SERVER_client_get_user_context_ (client, sizeof (type))
+
+/**
+ * Set user context to be associated with the given client.
+ *
+ * @param client client to query
+ * @param value pointer to user context
+ */
+#define GNUNET_SERVER_client_set_user_context(client,value)             \
+  GNUNET_SERVER_client_set_user_context_ (client, value, sizeof (*value))
+
+
+
+/**
+ * Notify us when the server has enough space to transmit
+ * a message of the given size to the given client.
+ *
+ * @param client client to transmit message to
+ * @param size requested amount of buffer space
+ * @param timeout after how long should we give up (and call
+ *        notify with buf NULL and size 0)?
+ * @param callback function to call when space is available
+ * @param callback_cls closure for @a callback
+ * @return non-NULL if the notify callback was queued; can be used
+ *           to cancel the request using
+ *           #GNUNET_SERVER_notify_transmit_ready_cancel.
+ *         NULL if we are already going to notify someone else (busy)
+ */
+struct GNUNET_SERVER_TransmitHandle *
+GNUNET_SERVER_notify_transmit_ready (struct GNUNET_SERVER_Client *client,
+                                     size_t size,
+                                     struct GNUNET_TIME_Relative timeout,
+                                     GNUNET_CONNECTION_TransmitReadyNotify callback,
+                                     void *callback_cls);
+
+/**
+ * Abort transmission request.
+ *
+ * @param th request to abort
+ */
+void
+GNUNET_SERVER_notify_transmit_ready_cancel (struct GNUNET_SERVER_TransmitHandle *th);
+
+
+
+
+/**
+ * Notify the server that the given client handle should
+ * be kept (keeps the connection up if possible, increments
+ * the internal reference counter).
+ *
+ * @param client the client to keep
+ */
+void
+GNUNET_SERVER_client_keep (struct GNUNET_SERVER_Client *client);
+
+
+/**
+ * Notify the server that the given client handle is no
+ * longer required.  Decrements the reference counter.  If
+ * that counter reaches zero an inactive connection maybe
+ * closed.
+ *
+ * @param client the client to drop
+ */
+void
+GNUNET_SERVER_client_drop (struct GNUNET_SERVER_Client *client);
+
+
+/**
+ * Function called by the service's run
+ * method to run service-specific setup code.
+ *
+ * @param cls closure
+ * @param server the initialized server
+ * @param cfg configuration to use
+ */
+typedef void
+(*LEGACY_SERVICE_Main) (void *cls,
+                        struct GNUNET_SERVER_Handle *server,
+                        const struct GNUNET_CONFIGURATION_Handle *cfg);
+
+
+
+/**
+ * Suspend accepting connections from the listen socket temporarily.
+ * Resume activity using #GNUNET_SERVER_resume.
+ *
+ * @param server server to stop accepting connections.
+ */
+void
+GNUNET_SERVER_suspend (struct GNUNET_SERVER_Handle *server);
+
+/**
+ * Notify us when the server has enough space to transmit
+ * a message of the given size to the given client.
+ *
+ * @param client client to transmit message to
+ * @param size requested amount of buffer space
+ * @param timeout after how long should we give up (and call
+ *        notify with buf NULL and size 0)?
+ * @param callback function to call when space is available
+ * @param callback_cls closure for @a callback
+ * @return non-NULL if the notify callback was queued; can be used
+ *           to cancel the request using
+ *           #GNUNET_SERVER_notify_transmit_ready_cancel.
+ *         NULL if we are already going to notify someone else (busy)
+ */
+struct GNUNET_SERVER_TransmitHandle *
+GNUNET_SERVER_notify_transmit_ready (struct GNUNET_SERVER_Client *client,
+                                     size_t size,
+                                     struct GNUNET_TIME_Relative timeout,
+                                     GNUNET_CONNECTION_TransmitReadyNotify callback,
+                                     void *callback_cls);
+
+
+/**
+ * Add a TCP socket-based connection to the set of handles managed by
+ * this server.  Use this function for outgoing (P2P) connections that
+ * we initiated (and where this server should process incoming
+ * messages).
+ *
+ * @param server the server to use
+ * @param connection the connection to manage (client must
+ *        stop using this connection from now on)
+ * @return the client handle
+ */
+struct GNUNET_SERVER_Client *
+GNUNET_SERVER_connect_socket (struct GNUNET_SERVER_Handle *server,
+                              struct GNUNET_CONNECTION_Handle *connection);
+
+
+/**
+ * Resume accepting connections from the listen socket.
+ *
+ * @param server server to resume accepting connections.
+ */
+void
+GNUNET_SERVER_resume (struct GNUNET_SERVER_Handle *server);
+
+/**
+ * Free resources held by this server.
+ *
+ * @param server server to destroy
+ */
+void
+GNUNET_SERVER_destroy (struct GNUNET_SERVER_Handle *server);
+
+
+
+
+#include "tcp_connection_legacy.c"
+#include "tcp_server_mst_legacy.c"
+#include "tcp_server_legacy.c"
+#include "tcp_service_legacy.c"
+
+GNUNET_NETWORK_STRUCT_BEGIN
+
+/**
  * Initial handshake message for a session.
  */
 struct WelcomeMessage
@@ -434,7 +962,7 @@ struct Plugin
   /**
    * Handle to the network service.
    */
-  struct GNUNET_SERVICE_Context *service;
+  struct LEGACY_SERVICE_Context *service;
 
   /**
    * Handle to the server for this service.
@@ -521,47 +1049,6 @@ struct Plugin
 };
 
 
-/* begin of ancient copy-and-pasted code that should be
-   specialized for TCP ...*/
-/**
- * Add the given UNIX domain path as an address to the
- * list (as the first entry).
- *
- * @param saddrs array to update
- * @param saddrlens where to store the address length
- * @param unixpath path to add
- * @param abstract #GNUNET_YES to add an abstract UNIX domain socket.  This
- *          parameter is ignore on systems other than LINUX
- */
-static void
-add_unixpath (struct sockaddr **saddrs,
-              socklen_t *saddrlens,
-              const char *unixpath,
-              int abstract)
-{
-#ifdef AF_UNIX
-  struct sockaddr_un *un;
-
-  un = GNUNET_new (struct sockaddr_un);
-  un->sun_family = AF_UNIX;
-  strncpy (un->sun_path, unixpath, sizeof (un->sun_path) - 1);
-#ifdef LINUX
-  if (GNUNET_YES == abstract)
-    un->sun_path[0] = '\0';
-#endif
-#if HAVE_SOCKADDR_UN_SUN_LEN
-  un->sun_len = (u_char) sizeof (struct sockaddr_un);
-#endif
-  *saddrs = (struct sockaddr *) un;
-  *saddrlens = sizeof (struct sockaddr_un);
-#else
-  /* this function should never be called
-   * unless AF_UNIX is defined! */
-  GNUNET_assert (0);
-#endif
-}
-
-
 /**
  * Get the list of addresses that a server for the given service
  * should bind to.
@@ -3289,7 +3776,7 @@ libgnunet_plugin_transport_tcp_init (void *cls)
   struct GNUNET_TRANSPORT_PluginEnvironment *env = cls;
   struct GNUNET_TRANSPORT_PluginFunctions *api;
   struct Plugin *plugin;
-  struct GNUNET_SERVICE_Context *service;
+  struct LEGACY_SERVICE_Context *service;
   unsigned long long aport;
   unsigned long long bport;
   unsigned long long max_connections;
@@ -3344,9 +3831,9 @@ libgnunet_plugin_transport_tcp_init (void *cls)
     aport = 0;
   if (0 != bport)
   {
-    service = GNUNET_SERVICE_start ("transport-tcp",
+    service = LEGACY_SERVICE_start ("transport-tcp",
                                     env->cfg,
-                                    GNUNET_SERVICE_OPTION_NONE);
+                                    LEGACY_SERVICE_OPTION_NONE);
     if (NULL == service)
     {
       LOG (GNUNET_ERROR_TYPE_WARNING,
@@ -3377,7 +3864,7 @@ libgnunet_plugin_transport_tcp_init (void *cls)
   {
 #ifdef TCP_STEALTH
     plugin->myoptions |= TCP_OPTIONS_TCP_STEALTH;
-    lsocks = GNUNET_SERVICE_get_listen_sockets (service);
+    lsocks = LEGACY_SERVICE_get_listen_sockets (service);
     if (NULL != lsocks)
     {
       uint32_t len = sizeof (struct WelcomeMessage);
@@ -3470,7 +3957,7 @@ libgnunet_plugin_transport_tcp_init (void *cls)
   plugin->service = service;
   if (NULL != service)
   {
-    plugin->server = GNUNET_SERVICE_get_server (service);
+    plugin->server = LEGACY_SERVICE_get_server (service);
   }
   else
   {
@@ -3533,7 +4020,7 @@ libgnunet_plugin_transport_tcp_init (void *cls)
     GNUNET_NAT_unregister (plugin->nat);
   GNUNET_CONTAINER_multipeermap_destroy (plugin->sessionmap);
   if (NULL != service)
-    GNUNET_SERVICE_stop (service);
+    LEGACY_SERVICE_stop (service);
   GNUNET_free (plugin);
   GNUNET_free_non_null (api);
   return NULL;
@@ -3586,7 +4073,7 @@ libgnunet_plugin_transport_tcp_done (void *cls)
   }
 
   if (NULL != plugin->service)
-    GNUNET_SERVICE_stop (plugin->service);
+    LEGACY_SERVICE_stop (plugin->service);
   else
     GNUNET_SERVER_destroy (plugin->server);
   GNUNET_free (plugin->handlers);
diff --git a/src/transport/plugin_transport_udp.c b/src/transport/plugin_transport_udp.c
index 76132523b..1ff962544 100644
--- a/src/transport/plugin_transport_udp.c
+++ b/src/transport/plugin_transport_udp.c
@@ -159,6 +159,11 @@ struct GNUNET_ATS_Session
   struct GNUNET_PeerIdentity target;
 
   /**
+   * Tokenizer for inbound messages.
+   */
+  struct GNUNET_MessageStreamTokenizer *mst;
+
+  /**
    * Plugin this session belongs to.
    */
   struct Plugin *plugin;
@@ -626,6 +631,11 @@ free_session (struct GNUNET_ATS_Session *s)
     GNUNET_free (s->frag_ctx);
     s->frag_ctx = NULL;
   }
+  if (NULL != s->mst)
+  {
+    GNUNET_MST_destroy (s->mst);
+    s->mst = NULL;
+  }
   GNUNET_free (s);
 }
 
@@ -1623,7 +1633,7 @@ enqueue (struct Plugin *plugin,
     GNUNET_break (0);
     return;
   }
-  if (plugin->bytes_in_buffer + udpw->msg_size > INT64_MAX)
+  if (plugin->bytes_in_buffer > INT64_MAX - udpw->msg_size)
   {
     GNUNET_break (0);
   }
@@ -2051,7 +2061,7 @@ udp_plugin_send (void *cls,
   if ( (sizeof(struct IPv4UdpAddress) == s->address->address_length) &&
        (NULL == plugin->sockv4) )
     return GNUNET_SYSERR;
-  if (udpmlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (udpmlen >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return GNUNET_SYSERR;
@@ -2499,18 +2509,16 @@ read_process_ack (struct Plugin *plugin,
  * Message tokenizer has broken up an incomming message. Pass it on
  * to the service.
  *
- * @param cls the `struct Plugin *`
- * @param client the `struct GNUNET_ATS_Session *`
+ * @param cls the `struct GNUNET_ATS_Session *`
  * @param hdr the actual message
  * @return #GNUNET_OK (always)
  */
 static int
 process_inbound_tokenized_messages (void *cls,
-                                    void *client,
                                     const struct GNUNET_MessageHeader *hdr)
 {
-  struct Plugin *plugin = cls;
-  struct GNUNET_ATS_Session *session = client;
+  struct GNUNET_ATS_Session *session = cls;
+  struct Plugin *plugin = session->plugin;
 
   if (GNUNET_YES == session->in_destroy)
     return GNUNET_OK;
@@ -2626,6 +2634,8 @@ udp_plugin_create_session (void *cls,
   struct GNUNET_ATS_Session *s;
 
   s = GNUNET_new (struct GNUNET_ATS_Session);
+  s->mst = GNUNET_MST_create (&process_inbound_tokenized_messages,
+                              s);
   s->plugin = plugin;
   s->address = GNUNET_HELLO_address_copy (address);
   s->target = address->peer;
@@ -2792,12 +2802,11 @@ process_udp_message (struct Plugin *plugin,
   GNUNET_free (address);
 
   s->rc++;
-  GNUNET_SERVER_mst_receive (plugin->mst,
-                             s,
-                             (const char *) &msg[1],
-                             ntohs (msg->header.size) - sizeof(struct UDPMessage),
-                             GNUNET_YES,
-                             GNUNET_NO);
+  GNUNET_MST_from_buffer (s->mst,
+                          (const char *) &msg[1],
+                          ntohs (msg->header.size) - sizeof(struct UDPMessage),
+                          GNUNET_YES,
+                          GNUNET_NO);
   s->rc--;
   if ( (0 == s->rc) &&
        (GNUNET_YES == s->in_destroy) )
@@ -3990,8 +3999,6 @@ libgnunet_plugin_transport_udp_init (void *cls)
   p->sessions = GNUNET_CONTAINER_multipeermap_create (16,
                                                       GNUNET_NO);
   p->defrag_ctxs = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
-  p->mst = GNUNET_SERVER_mst_create (&process_inbound_tokenized_messages,
-                                     p);
   GNUNET_BANDWIDTH_tracker_init (&p->tracker,
                                  NULL,
                                  NULL,
@@ -4008,7 +4015,6 @@ libgnunet_plugin_transport_udp_init (void *cls)
         _("Failed to create UDP network sockets\n"));
     GNUNET_CONTAINER_multipeermap_destroy (p->sessions);
     GNUNET_CONTAINER_heap_destroy (p->defrag_ctxs);
-    GNUNET_SERVER_mst_destroy (p->mst);
     if (NULL != p->nat)
       GNUNET_NAT_unregister (p->nat);
     GNUNET_free (p);
@@ -4120,11 +4126,6 @@ libgnunet_plugin_transport_udp_done (void *cls)
     GNUNET_CONTAINER_heap_destroy (plugin->defrag_ctxs);
     plugin->defrag_ctxs = NULL;
   }
-  if (NULL != plugin->mst)
-  {
-    GNUNET_SERVER_mst_destroy (plugin->mst);
-    plugin->mst = NULL;
-  }
   while (NULL != (udpw = plugin->ipv4_queue_head))
   {
     dequeue (plugin,
diff --git a/src/transport/plugin_transport_udp.h b/src/transport/plugin_transport_udp.h
index 152b16099..48c7365c7 100644
--- a/src/transport/plugin_transport_udp.h
+++ b/src/transport/plugin_transport_udp.h
@@ -164,11 +164,6 @@ struct Plugin
   struct GNUNET_SCHEDULER_Task *select_task_v6;
 
   /**
-   * Tokenizer for inbound messages.
-   */
-  struct GNUNET_SERVER_MessageStreamTokenizer *mst;
-
-  /**
    * Bandwidth tracker to limit global UDP traffic.
    */
   struct GNUNET_BANDWIDTH_Tracker tracker;
@@ -192,7 +187,7 @@ struct Plugin
    * Handle to NAT traversal support.
    */
   struct GNUNET_NAT_STUN_Handle *stun;
-  
+
   /**
    * The read socket for IPv4
    */
@@ -204,11 +199,6 @@ struct Plugin
   struct GNUNET_NETWORK_Handle *sockv6;
 
   /**
-   * Tokenizer for inbound messages.
-   */
-  struct GNUNET_SERVER_MessageStreamTokenizer *broadcast_mst;
-
-  /**
    * Head of DLL of broadcast addresses
    */
   struct BroadcastAddress *broadcast_tail;
diff --git a/src/transport/plugin_transport_udp_broadcasting.c b/src/transport/plugin_transport_udp_broadcasting.c
index a440830fd..c6ddbce9b 100644
--- a/src/transport/plugin_transport_udp_broadcasting.c
+++ b/src/transport/plugin_transport_udp_broadcasting.c
@@ -133,11 +133,10 @@ struct MstContext
  */
 static int
 broadcast_mst_cb (void *cls,
-                  void *client,
                   const struct GNUNET_MessageHeader *message)
 {
-  struct Plugin *plugin = cls;
-  struct MstContext *mc = client;
+  struct MstContext *mc = cls;
+  struct Plugin *plugin = mc->plugin;
   struct GNUNET_HELLO_Address *address;
   const struct GNUNET_MessageHeader *hello;
   const struct UDP_Beacon_Message *msg;
@@ -191,16 +190,20 @@ udp_broadcast_receive (struct Plugin *plugin,
                        size_t udp_addr_len,
                        enum GNUNET_ATS_Network_Type network_type)
 {
+  struct GNUNET_MessageStreamTokenizer *broadcast_mst;
   struct MstContext mc;
 
+  broadcast_mst = GNUNET_MST_create (&broadcast_mst_cb,
+                                     &mc);
+  mc.plugin = plugin;
   mc.udp_addr = udp_addr;
   mc.udp_addr_len = udp_addr_len;
   mc.ats_address_network_type = network_type;
-  GNUNET_SERVER_mst_receive (plugin->broadcast_mst,
-                             &mc,
-                             buf, size,
-                             GNUNET_NO,
-                             GNUNET_NO);
+  GNUNET_MST_from_buffer (broadcast_mst,
+                          buf, size,
+                          GNUNET_NO,
+                          GNUNET_NO);
+  GNUNET_MST_destroy (broadcast_mst);
 }
 
 
@@ -546,10 +549,6 @@ setup_broadcast (struct Plugin *plugin,
     return;
   }
 
-  /* always create tokenizers */
-  plugin->broadcast_mst =
-    GNUNET_SERVER_mst_create (&broadcast_mst_cb, plugin);
-
   if (GNUNET_YES != plugin->enable_broadcasting)
     return; /* We do not send, just receive */
 
@@ -636,13 +635,6 @@ stop_broadcast (struct Plugin *plugin)
       GNUNET_free (p);
     }
   }
-
-  /* Destroy MSTs */
-  if (NULL != plugin->broadcast_mst)
-  {
-    GNUNET_SERVER_mst_destroy (plugin->broadcast_mst);
-    plugin->broadcast_mst = NULL;
-  }
 }
 
 /* end of plugin_transport_udp_broadcasting.c */
diff --git a/src/transport/plugin_transport_wlan.c b/src/transport/plugin_transport_wlan.c
index 376065d24..b23739d3c 100644
--- a/src/transport/plugin_transport_wlan.c
+++ b/src/transport/plugin_transport_wlan.c
@@ -38,6 +38,7 @@
 #include "gnunet_fragmentation_lib.h"
 #include "gnunet_constants.h"
 
+
 #if BUILD_WLAN
 /* begin case wlan */
 #define PLUGIN_NAME "wlan"
@@ -48,6 +49,7 @@
 #define LIBGNUNET_PLUGIN_TRANSPORT_DONE libgnunet_plugin_transport_wlan_done
 #define LOG(kind,...) GNUNET_log_from (kind, "transport-wlan",__VA_ARGS__)
 
+
 /**
  * time out of a mac endpoint
  */
@@ -92,6 +94,30 @@
 #error need to build wlan or bluetooth
 #endif
 
+
+
+/**
+ * Functions with this signature are called whenever a
+ * complete message is received by the tokenizer.
+ *
+ * Do not call #GNUNET_SERVER_mst_destroy from within
+ * the scope of this callback.
+ *
+ * @param cls closure
+ * @param client identification of the client
+ * @param message the actual message
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR to stop further processing
+ */
+typedef int
+(*GNUNET_SERVER_MessageTokenizerCallback) (void *cls,
+                                           void *client,
+                                           const struct GNUNET_MessageHeader *message);
+
+
+/* Include legacy message stream tokenizer that was removed from util (for now) */
+#include "tcp_server_mst_legacy.c"
+
+
 /**
  * Max size of packet (that we give to the WLAN driver for transmission)
  */
@@ -718,7 +744,7 @@ send_ack (void *cls,
     GNUNET_break (0);
     return;
   }
-  if (size >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+  if (size >= GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return;
@@ -1728,11 +1754,10 @@ send_hello_beacon (void *cls)
  * Function used for to process the data from the suid process
  *
  * @param cls the plugin handle
- * @param client client that send the data (not used)
  * @param hdr header of the GNUNET_MessageHeader
  */
 static int
-handle_helper_message (void *cls, void *client,
+handle_helper_message (void *cls,
                        const struct GNUNET_MessageHeader *hdr)
 {
   struct Plugin *plugin = cls;
diff --git a/src/util/connection.c b/src/transport/tcp_connection_legacy.c
index e822b264f..5b219a467 100644
--- a/src/util/connection.c
+++ b/src/transport/tcp_connection_legacy.c
@@ -35,7 +35,15 @@
 #include "gnunet_resolver_service.h"
 
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util-connection", __VA_ARGS__)
+/**
+ * Timeout we use on TCP connect before trying another
+ * result from the DNS resolver.  Actual value used
+ * is this value divided by the number of address families.
+ * Default is 5s.
+ */
+#define CONNECT_RETRY_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
+
+
 
 #define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-connection", syscall)
 
@@ -305,7 +313,7 @@ GNUNET_CONNECTION_create_from_existing (struct GNUNET_NETWORK_Handle *osSocket)
   struct GNUNET_CONNECTION_Handle *connection;
 
   connection = GNUNET_new (struct GNUNET_CONNECTION_Handle);
-  connection->write_buffer_size = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  connection->write_buffer_size = GNUNET_MIN_MESSAGE_SIZE;
   connection->write_buffer = GNUNET_malloc (connection->write_buffer_size);
   connection->sock = osSocket;
   return connection;
@@ -452,7 +460,7 @@ GNUNET_CONNECTION_create_from_accept (GNUNET_CONNECTION_AccessCheck access_cb,
     return NULL;
   }
   connection = GNUNET_new (struct GNUNET_CONNECTION_Handle);
-  connection->write_buffer_size = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  connection->write_buffer_size = GNUNET_MIN_MESSAGE_SIZE;
   connection->write_buffer = GNUNET_malloc (connection->write_buffer_size);
   connection->addr = uaddr;
   connection->addrlen = addrlen;
@@ -825,7 +833,7 @@ try_connect_using_address (void *cls,
     return;
   }
   GNUNET_CONTAINER_DLL_insert (connection->ap_head, connection->ap_tail, ap);
-  delay = GNUNET_CONNECTION_CONNECT_RETRY_TIMEOUT;
+  delay = CONNECT_RETRY_TIMEOUT;
   if (NULL != connection->nth.notify_ready)
     delay = GNUNET_TIME_relative_min (delay,
                                       GNUNET_TIME_absolute_get_remaining (connection->nth.transmit_timeout));
@@ -859,14 +867,14 @@ GNUNET_CONNECTION_create_from_connect (const struct GNUNET_CONFIGURATION_Handle
   GNUNET_assert (0 < strlen (hostname));        /* sanity check */
   connection = GNUNET_new (struct GNUNET_CONNECTION_Handle);
   connection->cfg = cfg;
-  connection->write_buffer_size = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  connection->write_buffer_size = GNUNET_MIN_MESSAGE_SIZE;
   connection->write_buffer = GNUNET_malloc (connection->write_buffer_size);
   connection->port = port;
   connection->hostname = GNUNET_strdup (hostname);
   connection->dns_active =
       GNUNET_RESOLVER_ip_get (connection->hostname,
                               AF_UNSPEC,
-                              GNUNET_CONNECTION_CONNECT_RETRY_TIMEOUT,
+                              CONNECT_RETRY_TIMEOUT,
                               &try_connect_using_address,
                               connection);
   return connection;
@@ -910,7 +918,7 @@ GNUNET_CONNECTION_create_from_connect_to_unixpath (const struct GNUNET_CONFIGURA
 #endif
   connection = GNUNET_new (struct GNUNET_CONNECTION_Handle);
   connection->cfg = cfg;
-  connection->write_buffer_size = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  connection->write_buffer_size = GNUNET_MIN_MESSAGE_SIZE;
   connection->write_buffer = GNUNET_malloc (connection->write_buffer_size);
   connection->port = 0;
   connection->hostname = NULL;
@@ -1536,7 +1544,7 @@ GNUNET_CONNECTION_notify_transmit_ready (struct GNUNET_CONNECTION_Handle *connec
     return NULL;
   }
   GNUNET_assert (NULL != notify);
-  GNUNET_assert (size < GNUNET_SERVER_MAX_MESSAGE_SIZE);
+  GNUNET_assert (size < GNUNET_MAX_MESSAGE_SIZE);
   GNUNET_assert (connection->write_buffer_off <= connection->write_buffer_size);
   GNUNET_assert (connection->write_buffer_pos <= connection->write_buffer_size);
   GNUNET_assert (connection->write_buffer_pos <= connection->write_buffer_off);
diff --git a/src/util/server.c b/src/transport/tcp_server_legacy.c
index 83c30e328..6b4daa525 100644
--- a/src/util/server.c
+++ b/src/transport/tcp_server_legacy.c
@@ -28,10 +28,6 @@
 #include "gnunet_util_lib.h"
 #include "gnunet_protocols.h"
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util-server", __VA_ARGS__)
-
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-server", syscall)
-
 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-server", syscall, filename)
 
 
@@ -1049,7 +1045,7 @@ process_mst (struct GNUNET_SERVER_Client *client,
            GNUNET_STRINGS_relative_time_to_string (client->idle_timeout, GNUNET_YES));
       client->receive_pending = GNUNET_YES;
       GNUNET_CONNECTION_receive (client->connection,
-                                 GNUNET_SERVER_MAX_MESSAGE_SIZE - 1,
+                                 GNUNET_MAX_MESSAGE_SIZE - 1,
                                  client->idle_timeout,
                                  &process_incoming,
                                  client);
@@ -1128,7 +1124,7 @@ process_incoming (void *cls,
          client);
     client->receive_pending = GNUNET_YES;
     GNUNET_CONNECTION_receive (client->connection,
-                               GNUNET_SERVER_MAX_MESSAGE_SIZE - 1,
+                               GNUNET_MAX_MESSAGE_SIZE - 1,
                                GNUNET_TIME_absolute_get_remaining (end),
                                &process_incoming,
                                client);
@@ -1204,7 +1200,7 @@ restart_processing (void *cls)
     LOG (GNUNET_ERROR_TYPE_DEBUG, "Server begins to read again from client.\n");
     client->receive_pending = GNUNET_YES;
     GNUNET_CONNECTION_receive (client->connection,
-                               GNUNET_SERVER_MAX_MESSAGE_SIZE - 1,
+                               GNUNET_MAX_MESSAGE_SIZE - 1,
                                client->idle_timeout,
                                &process_incoming,
                                client);
@@ -1292,7 +1288,7 @@ GNUNET_SERVER_connect_socket (struct GNUNET_SERVER_Handle *server,
     n->callback (n->callback_cls, client);
   client->receive_pending = GNUNET_YES;
   GNUNET_CONNECTION_receive (client->connection,
-                             GNUNET_SERVER_MAX_MESSAGE_SIZE - 1,
+                             GNUNET_MAX_MESSAGE_SIZE - 1,
                              client->idle_timeout,
                              &process_incoming,
                              client);
diff --git a/src/util/server_mst.c b/src/transport/tcp_server_mst_legacy.c
index 5155b54da..78e04c1b1 100644
--- a/src/util/server_mst.c
+++ b/src/transport/tcp_server_mst_legacy.c
@@ -34,8 +34,6 @@
 #define ALIGN_FACTOR 8
 #endif
 
-#define LOG(kind,...) GNUNET_log_from (kind, "util-server-mst", __VA_ARGS__)
-
 
 /**
  * Handle to a message stream tokenizer.
@@ -91,8 +89,8 @@ GNUNET_SERVER_mst_create (GNUNET_SERVER_MessageTokenizerCallback cb,
   struct GNUNET_SERVER_MessageStreamTokenizer *ret;
 
   ret = GNUNET_new (struct GNUNET_SERVER_MessageStreamTokenizer);
-  ret->hdr = GNUNET_malloc (GNUNET_SERVER_MIN_BUFFER_SIZE);
-  ret->curr_buf = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  ret->hdr = GNUNET_malloc (GNUNET_MIN_MESSAGE_SIZE);
+  ret->curr_buf = GNUNET_MIN_MESSAGE_SIZE;
   ret->cb = cb;
   ret->cb_cls = cb_cls;
   return ret;
diff --git a/src/transport/tcp_service_legacy.c b/src/transport/tcp_service_legacy.c
new file mode 100644
index 000000000..7c2d3e55a
--- /dev/null
+++ b/src/transport/tcp_service_legacy.c
@@ -0,0 +1,1688 @@
+/*
+     This file is part of GNUnet.
+     Copyright (C) 2009, 2012 GNUnet e.V.
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+     Boston, MA 02110-1301, USA.
+*/
+
+/**
+ * @file util/service.c
+ * @brief functions related to starting services
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "gnunet_util_lib.h"
+#include "gnunet_protocols.h"
+#include "gnunet_constants.h"
+#include "gnunet_resolver_service.h"
+
+#if HAVE_MALLINFO
+#include <malloc.h>
+#include "gauger.h"
+#endif
+
+
+/* ******************* access control ******************** */
+
+/**
+ * Check if the given IP address is in the list of IP addresses.
+ *
+ * @param list a list of networks
+ * @param add the IP to check (in network byte order)
+ * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
+ */
+static int
+check_ipv4_listed (const struct GNUNET_STRINGS_IPv4NetworkPolicy *list,
+                   const struct in_addr *add)
+{
+  unsigned int i;
+
+  if (NULL == list)
+    return GNUNET_NO;
+  i = 0;
+  while ((list[i].network.s_addr != 0) || (list[i].netmask.s_addr != 0))
+  {
+    if ((add->s_addr & list[i].netmask.s_addr) ==
+        (list[i].network.s_addr & list[i].netmask.s_addr))
+      return GNUNET_YES;
+    i++;
+  }
+  return GNUNET_NO;
+}
+
+
+/**
+ * Check if the given IP address is in the list of IP addresses.
+ *
+ * @param list a list of networks
+ * @param ip the IP to check (in network byte order)
+ * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
+ */
+static int
+check_ipv6_listed (const struct GNUNET_STRINGS_IPv6NetworkPolicy *list,
+                   const struct in6_addr *ip)
+{
+  unsigned int i;
+  unsigned int j;
+  struct in6_addr zero;
+
+  if (NULL == list)
+    return GNUNET_NO;
+  memset (&zero, 0, sizeof (struct in6_addr));
+  i = 0;
+NEXT:
+  while (0 != memcmp (&zero, &list[i].network, sizeof (struct in6_addr)))
+  {
+    for (j = 0; j < sizeof (struct in6_addr) / sizeof (int); j++)
+      if (((((int *) ip)[j] & ((int *) &list[i].netmask)[j])) !=
+          (((int *) &list[i].network)[j] & ((int *) &list[i].netmask)[j]))
+      {
+        i++;
+        goto NEXT;
+      }
+    return GNUNET_YES;
+  }
+  return GNUNET_NO;
+}
+
+
+/* ****************** service struct ****************** */
+
+
+/**
+ * Context for "service_task".
+ */
+struct LEGACY_SERVICE_Context
+{
+  /**
+   * Our configuration.
+   */
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
+
+  /**
+   * Handle for the server.
+   */
+  struct GNUNET_SERVER_Handle *server;
+
+  /**
+   * NULL-terminated array of addresses to bind to, NULL if we got pre-bound
+   * listen sockets.
+   */
+  struct sockaddr **addrs;
+
+  /**
+   * Name of our service.
+   */
+  const char *service_name;
+
+  /**
+   * Main service-specific task to run.
+   */
+  LEGACY_SERVICE_Main task;
+
+  /**
+   * Closure for @e task.
+   */
+  void *task_cls;
+
+  /**
+   * IPv4 addresses that are not allowed to connect.
+   */
+  struct GNUNET_STRINGS_IPv4NetworkPolicy *v4_denied;
+
+  /**
+   * IPv6 addresses that are not allowed to connect.
+   */
+  struct GNUNET_STRINGS_IPv6NetworkPolicy *v6_denied;
+
+  /**
+   * IPv4 addresses that are allowed to connect (if not
+   * set, all are allowed).
+   */
+  struct GNUNET_STRINGS_IPv4NetworkPolicy *v4_allowed;
+
+  /**
+   * IPv6 addresses that are allowed to connect (if not
+   * set, all are allowed).
+   */
+  struct GNUNET_STRINGS_IPv6NetworkPolicy *v6_allowed;
+
+  /**
+   * My (default) message handlers.  Adjusted copy
+   * of "defhandlers".
+   */
+  struct GNUNET_SERVER_MessageHandler *my_handlers;
+
+  /**
+   * Array of the lengths of the entries in addrs.
+   */
+  socklen_t *addrlens;
+
+  /**
+   * NULL-terminated array of listen sockets we should take over.
+   */
+  struct GNUNET_NETWORK_Handle **lsocks;
+
+  /**
+   * Task ID of the shutdown task.
+   */
+  struct GNUNET_SCHEDULER_Task *shutdown_task;
+
+  /**
+   * Idle timeout for server.
+   */
+  struct GNUNET_TIME_Relative timeout;
+
+  /**
+   * Overall success/failure of the service start.
+   */
+  int ret;
+
+  /**
+   * If we are daemonizing, this FD is set to the
+   * pipe to the parent.  Send '.' if we started
+   * ok, '!' if not.  -1 if we are not daemonizing.
+   */
+  int ready_confirm_fd;
+
+  /**
+   * Do we close connections if we receive messages
+   * for which we have no handler?
+   */
+  int require_found;
+
+  /**
+   * Do we require a matching UID for UNIX domain socket connections?
+   * #GNUNET_NO means that the UID does not have to match (however,
+   * @e match_gid may still impose other access control checks).
+   */
+  int match_uid;
+
+  /**
+   * Do we require a matching GID for UNIX domain socket connections?
+   * Ignored if @e match_uid is #GNUNET_YES.  Note that this is about
+   * checking that the client's UID is in our group OR that the
+   * client's GID is our GID.  If both "match_gid" and @e match_uid are
+   * #GNUNET_NO, all users on the local system have access.
+   */
+  int match_gid;
+
+  /**
+   * Our options.
+   */
+  enum LEGACY_SERVICE_Options options;
+
+};
+
+
+/* ****************** message handlers ****************** */
+
+/**
+ * Send a 'TEST' message back to the client.
+ *
+ * @param cls the 'struct GNUNET_SERVER_Client' to send TEST to
+ * @param size number of bytes available in 'buf'
+ * @param buf where to copy the message
+ * @return number of bytes written to 'buf'
+ */
+static size_t
+write_test (void *cls, size_t size, void *buf)
+{
+  struct GNUNET_SERVER_Client *client = cls;
+  struct GNUNET_MessageHeader *msg;
+
+  if (size < sizeof (struct GNUNET_MessageHeader))
+  {
+    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+    return 0;                   /* client disconnected */
+  }
+  msg = (struct GNUNET_MessageHeader *) buf;
+  msg->type = htons (GNUNET_MESSAGE_TYPE_TEST);
+  msg->size = htons (sizeof (struct GNUNET_MessageHeader));
+  GNUNET_SERVER_receive_done (client, GNUNET_OK);
+  return sizeof (struct GNUNET_MessageHeader);
+}
+
+
+/**
+ * Handler for TEST message.
+ *
+ * @param cls closure (refers to service)
+ * @param client identification of the client
+ * @param message the actual message
+ */
+static void
+handle_test (void *cls, struct GNUNET_SERVER_Client *client,
+             const struct GNUNET_MessageHeader *message)
+{
+  /* simply bounce message back to acknowledge */
+  if (NULL ==
+      GNUNET_SERVER_notify_transmit_ready (client,
+                                           sizeof (struct GNUNET_MessageHeader),
+                                           GNUNET_TIME_UNIT_FOREVER_REL,
+                                           &write_test, client))
+    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+}
+
+
+/**
+ * Default handlers for all services.  Will be copied and the
+ * "callback_cls" fields will be replaced with the specific service
+ * struct.
+ */
+static const struct GNUNET_SERVER_MessageHandler defhandlers[] = {
+  {&handle_test, NULL, GNUNET_MESSAGE_TYPE_TEST,
+   sizeof (struct GNUNET_MessageHeader)},
+  {NULL, NULL, 0, 0}
+};
+
+
+/* ****************** service core routines ************** */
+
+
+/**
+ * Check if access to the service is allowed from the given address.
+ *
+ * @param cls closure
+ * @param uc credentials, if available, otherwise NULL
+ * @param addr address
+ * @param addrlen length of address
+ * @return #GNUNET_YES to allow, #GNUNET_NO to deny, #GNUNET_SYSERR
+ *   for unknown address family (will be denied).
+ */
+static int
+check_access (void *cls, const struct GNUNET_CONNECTION_Credentials *uc,
+              const struct sockaddr *addr, socklen_t addrlen)
+{
+  struct LEGACY_SERVICE_Context *sctx = cls;
+  const struct sockaddr_in *i4;
+  const struct sockaddr_in6 *i6;
+  int ret;
+
+  switch (addr->sa_family)
+  {
+  case AF_INET:
+    GNUNET_assert (addrlen == sizeof (struct sockaddr_in));
+    i4 = (const struct sockaddr_in *) addr;
+    ret = ((NULL == sctx->v4_allowed) ||
+           (check_ipv4_listed (sctx->v4_allowed, &i4->sin_addr))) &&
+        ((NULL == sctx->v4_denied) ||
+         (!check_ipv4_listed (sctx->v4_denied, &i4->sin_addr)));
+    break;
+  case AF_INET6:
+    GNUNET_assert (addrlen == sizeof (struct sockaddr_in6));
+    i6 = (const struct sockaddr_in6 *) addr;
+    ret = ((NULL == sctx->v6_allowed) ||
+           (check_ipv6_listed (sctx->v6_allowed, &i6->sin6_addr))) &&
+        ((NULL == sctx->v6_denied) ||
+         (!check_ipv6_listed (sctx->v6_denied, &i6->sin6_addr)));
+    break;
+#ifndef WINDOWS
+  case AF_UNIX:
+    ret = GNUNET_OK;            /* controlled using file-system ACL now */
+    break;
+#endif
+  default:
+    LOG (GNUNET_ERROR_TYPE_WARNING, _("Unknown address family %d\n"),
+         addr->sa_family);
+    return GNUNET_SYSERR;
+  }
+  if (GNUNET_OK != ret)
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         _("Access from `%s' denied to service `%s'\n"),
+         GNUNET_a2s (addr, addrlen),
+         sctx->service_name);
+  }
+  return ret;
+}
+
+
+/**
+ * Get the name of the file where we will
+ * write the PID of the service.
+ *
+ * @param sctx service context
+ * @return name of the file for the process ID
+ */
+static char *
+get_pid_file_name (struct LEGACY_SERVICE_Context *sctx)
+{
+  char *pif;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->service_name,
+                                               "PIDFILE", &pif))
+    return NULL;
+  return pif;
+}
+
+
+/**
+ * Parse an IPv4 access control list.
+ *
+ * @param ret location where to write the ACL (set)
+ * @param sctx service context to use to get the configuration
+ * @param option name of the ACL option to parse
+ * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
+ *         no ACL configured)
+ */
+static int
+process_acl4 (struct GNUNET_STRINGS_IPv4NetworkPolicy **ret,
+              struct LEGACY_SERVICE_Context *sctx,
+              const char *option)
+{
+  char *opt;
+
+  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, option))
+  {
+    *ret = NULL;
+    return GNUNET_OK;
+  }
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_CONFIGURATION_get_value_string (sctx->cfg,
+                                                       sctx->service_name,
+                                                       option, &opt));
+  if (NULL == (*ret = GNUNET_STRINGS_parse_ipv4_policy (opt)))
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         _("Could not parse IPv4 network specification `%s' for `%s:%s'\n"),
+         opt, sctx->service_name, option);
+    GNUNET_free (opt);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (opt);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Parse an IPv6 access control list.
+ *
+ * @param ret location where to write the ACL (set)
+ * @param sctx service context to use to get the configuration
+ * @param option name of the ACL option to parse
+ * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
+ *         no ACL configured)
+ */
+static int
+process_acl6 (struct GNUNET_STRINGS_IPv6NetworkPolicy **ret,
+              struct LEGACY_SERVICE_Context *sctx,
+              const char *option)
+{
+  char *opt;
+
+  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, option))
+  {
+    *ret = NULL;
+    return GNUNET_OK;
+  }
+  GNUNET_break (GNUNET_OK ==
+                GNUNET_CONFIGURATION_get_value_string (sctx->cfg,
+                                                       sctx->service_name,
+                                                       option, &opt));
+  if (NULL == (*ret = GNUNET_STRINGS_parse_ipv6_policy (opt)))
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         _("Could not parse IPv6 network specification `%s' for `%s:%s'\n"),
+         opt, sctx->service_name, option);
+    GNUNET_free (opt);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (opt);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Add the given UNIX domain path as an address to the
+ * list (as the first entry).
+ *
+ * @param saddrs array to update
+ * @param saddrlens where to store the address length
+ * @param unixpath path to add
+ * @param abstract #GNUNET_YES to add an abstract UNIX domain socket.  This
+ *          parameter is ignore on systems other than LINUX
+ */
+static void
+add_unixpath (struct sockaddr **saddrs,
+              socklen_t *saddrlens,
+              const char *unixpath,
+              int abstract)
+{
+#ifdef AF_UNIX
+  struct sockaddr_un *un;
+
+  un = GNUNET_new (struct sockaddr_un);
+  un->sun_family = AF_UNIX;
+  strncpy (un->sun_path, unixpath, sizeof (un->sun_path) - 1);
+#ifdef LINUX
+  if (GNUNET_YES == abstract)
+    un->sun_path[0] = '\0';
+#endif
+#if HAVE_SOCKADDR_UN_SUN_LEN
+  un->sun_len = (u_char) sizeof (struct sockaddr_un);
+#endif
+  *saddrs = (struct sockaddr *) un;
+  *saddrlens = sizeof (struct sockaddr_un);
+#else
+  /* this function should never be called
+   * unless AF_UNIX is defined! */
+  GNUNET_assert (0);
+#endif
+}
+
+
+/**
+ * Get the list of addresses that a server for the given service
+ * should bind to.
+ *
+ * @param service_name name of the service
+ * @param cfg configuration (which specifies the addresses)
+ * @param addrs set (call by reference) to an array of pointers to the
+ *              addresses the server should bind to and listen on; the
+ *              array will be NULL-terminated (on success)
+ * @param addr_lens set (call by reference) to an array of the lengths
+ *              of the respective `struct sockaddr` struct in the @a addrs
+ *              array (on success)
+ * @return number of addresses found on success,
+ *              #GNUNET_SYSERR if the configuration
+ *              did not specify reasonable finding information or
+ *              if it specified a hostname that could not be resolved;
+ *              #GNUNET_NO if the number of addresses configured is
+ *              zero (in this case, `*addrs` and `*addr_lens` will be
+ *              set to NULL).
+ */
+int
+LEGACY_SERVICE_get_server_addresses (const char *service_name,
+                                     const struct GNUNET_CONFIGURATION_Handle *cfg,
+                                     struct sockaddr ***addrs,
+                                     socklen_t ** addr_lens)
+{
+  int disablev6;
+  struct GNUNET_NETWORK_Handle *desc;
+  unsigned long long port;
+  char *unixpath;
+  struct addrinfo hints;
+  struct addrinfo *res;
+  struct addrinfo *pos;
+  struct addrinfo *next;
+  unsigned int i;
+  int resi;
+  int ret;
+  int abstract;
+  struct sockaddr **saddrs;
+  socklen_t *saddrlens;
+  char *hostname;
+
+  *addrs = NULL;
+  *addr_lens = NULL;
+  desc = NULL;
+  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "DISABLEV6"))
+  {
+    if (GNUNET_SYSERR ==
+        (disablev6 =
+         GNUNET_CONFIGURATION_get_value_yesno (cfg, service_name, "DISABLEV6")))
+      return GNUNET_SYSERR;
+  }
+  else
+    disablev6 = GNUNET_NO;
+
+  if (! disablev6)
+  {
+    /* probe IPv6 support */
+    desc = GNUNET_NETWORK_socket_create (PF_INET6, SOCK_STREAM, 0);
+    if (NULL == desc)
+    {
+      if ((ENOBUFS == errno) || (ENOMEM == errno) || (ENFILE == errno) ||
+          (EACCES == errno))
+      {
+        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "socket");
+        return GNUNET_SYSERR;
+      }
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Disabling IPv6 support for service `%s', failed to create IPv6 socket: %s\n"),
+           service_name, STRERROR (errno));
+      disablev6 = GNUNET_YES;
+    }
+    else
+    {
+      GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (desc));
+      desc = NULL;
+    }
+  }
+
+  port = 0;
+  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "PORT"))
+  {
+    if (GNUNET_OK !=
+        GNUNET_CONFIGURATION_get_value_number (cfg, service_name,
+                                               "PORT", &port))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Require valid port number for service `%s' in configuration!\n"),
+           service_name);
+    }
+    if (port > 65535)
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Require valid port number for service `%s' in configuration!\n"),
+           service_name);
+      return GNUNET_SYSERR;
+    }
+  }
+
+  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "BINDTO"))
+  {
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_CONFIGURATION_get_value_string (cfg, service_name,
+                                                         "BINDTO", &hostname));
+  }
+  else
+    hostname = NULL;
+
+  unixpath = NULL;
+  abstract = GNUNET_NO;
+#ifdef AF_UNIX
+  if ((GNUNET_YES ==
+       GNUNET_CONFIGURATION_have_value (cfg, service_name, "UNIXPATH")) &&
+      (GNUNET_OK ==
+       GNUNET_CONFIGURATION_get_value_filename (cfg, service_name, "UNIXPATH",
+                                              &unixpath)) &&
+      (0 < strlen (unixpath)))
+  {
+    /* probe UNIX support */
+    struct sockaddr_un s_un;
+
+    if (strlen (unixpath) >= sizeof (s_un.sun_path))
+    {
+      LOG (GNUNET_ERROR_TYPE_WARNING,
+           _("UNIXPATH `%s' too long, maximum length is %llu\n"), unixpath,
+           (unsigned long long) sizeof (s_un.sun_path));
+      unixpath = GNUNET_NETWORK_shorten_unixpath (unixpath);
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Using `%s' instead\n"),
+           unixpath);
+    }
+#ifdef LINUX
+    abstract = GNUNET_CONFIGURATION_get_value_yesno (cfg,
+                                                     "TESTING",
+                                                     "USE_ABSTRACT_SOCKETS");
+    if (GNUNET_SYSERR == abstract)
+      abstract = GNUNET_NO;
+#endif
+    if ((GNUNET_YES != abstract)
+        && (GNUNET_OK !=
+            GNUNET_DISK_directory_create_for_file (unixpath)))
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                                "mkdir",
+                                unixpath);
+  }
+  if (NULL != unixpath)
+  {
+    desc = GNUNET_NETWORK_socket_create (AF_UNIX, SOCK_STREAM, 0);
+    if (NULL == desc)
+    {
+      if ((ENOBUFS == errno) || (ENOMEM == errno) || (ENFILE == errno) ||
+          (EACCES == errno))
+      {
+        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "socket");
+        GNUNET_free_non_null (hostname);
+        GNUNET_free (unixpath);
+        return GNUNET_SYSERR;
+      }
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Disabling UNIX domain socket support for service `%s', failed to create UNIX domain socket: %s\n"),
+           service_name,
+           STRERROR (errno));
+      GNUNET_free (unixpath);
+      unixpath = NULL;
+    }
+    else
+    {
+      GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (desc));
+      desc = NULL;
+    }
+  }
+#endif
+
+  if ((0 == port) && (NULL == unixpath))
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Have neither PORT nor UNIXPATH for service `%s', but one is required\n"),
+         service_name);
+    GNUNET_free_non_null (hostname);
+    return GNUNET_SYSERR;
+  }
+  if (0 == port)
+  {
+    saddrs = GNUNET_malloc (2 * sizeof (struct sockaddr *));
+    saddrlens = GNUNET_malloc (2 * sizeof (socklen_t));
+    add_unixpath (saddrs, saddrlens, unixpath, abstract);
+    GNUNET_free_non_null (unixpath);
+    GNUNET_free_non_null (hostname);
+    *addrs = saddrs;
+    *addr_lens = saddrlens;
+    return 1;
+  }
+
+  if (NULL != hostname)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Resolving `%s' since that is where `%s' will bind to.\n",
+         hostname,
+         service_name);
+    memset (&hints, 0, sizeof (struct addrinfo));
+    if (disablev6)
+      hints.ai_family = AF_INET;
+    hints.ai_protocol = IPPROTO_TCP;
+    if ((0 != (ret = getaddrinfo (hostname, NULL, &hints, &res))) ||
+        (NULL == res))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Failed to resolve `%s': %s\n"),
+           hostname,
+           gai_strerror (ret));
+      GNUNET_free (hostname);
+      GNUNET_free_non_null (unixpath);
+      return GNUNET_SYSERR;
+    }
+    next = res;
+    i = 0;
+    while (NULL != (pos = next))
+    {
+      next = pos->ai_next;
+      if ((disablev6) && (pos->ai_family == AF_INET6))
+        continue;
+      i++;
+    }
+    if (0 == i)
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Failed to find %saddress for `%s'.\n"),
+           disablev6 ? "IPv4 " : "",
+           hostname);
+      freeaddrinfo (res);
+      GNUNET_free (hostname);
+      GNUNET_free_non_null (unixpath);
+      return GNUNET_SYSERR;
+    }
+    resi = i;
+    if (NULL != unixpath)
+      resi++;
+    saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
+    saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+    i = 0;
+    if (NULL != unixpath)
+    {
+      add_unixpath (saddrs, saddrlens, unixpath, abstract);
+      i++;
+    }
+    next = res;
+    while (NULL != (pos = next))
+    {
+      next = pos->ai_next;
+      if ((disablev6) && (AF_INET6 == pos->ai_family))
+        continue;
+      if ((IPPROTO_TCP != pos->ai_protocol) && (0 != pos->ai_protocol))
+        continue;               /* not TCP */
+      if ((SOCK_STREAM != pos->ai_socktype) && (0 != pos->ai_socktype))
+        continue;               /* huh? */
+      LOG (GNUNET_ERROR_TYPE_DEBUG, "Service `%s' will bind to `%s'\n",
+           service_name, GNUNET_a2s (pos->ai_addr, pos->ai_addrlen));
+      if (AF_INET == pos->ai_family)
+      {
+        GNUNET_assert (sizeof (struct sockaddr_in) == pos->ai_addrlen);
+        saddrlens[i] = pos->ai_addrlen;
+        saddrs[i] = GNUNET_malloc (saddrlens[i]);
+        GNUNET_memcpy (saddrs[i], pos->ai_addr, saddrlens[i]);
+        ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
+      }
+      else
+      {
+        GNUNET_assert (AF_INET6 == pos->ai_family);
+        GNUNET_assert (sizeof (struct sockaddr_in6) == pos->ai_addrlen);
+        saddrlens[i] = pos->ai_addrlen;
+        saddrs[i] = GNUNET_malloc (saddrlens[i]);
+        GNUNET_memcpy (saddrs[i], pos->ai_addr, saddrlens[i]);
+        ((struct sockaddr_in6 *) saddrs[i])->sin6_port = htons (port);
+      }
+      i++;
+    }
+    GNUNET_free (hostname);
+    freeaddrinfo (res);
+    resi = i;
+  }
+  else
+  {
+    /* will bind against everything, just set port */
+    if (disablev6)
+    {
+      /* V4-only */
+      resi = 1;
+      if (NULL != unixpath)
+        resi++;
+      i = 0;
+      saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
+      saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+      if (NULL != unixpath)
+      {
+        add_unixpath (saddrs, saddrlens, unixpath, abstract);
+        i++;
+      }
+      saddrlens[i] = sizeof (struct sockaddr_in);
+      saddrs[i] = GNUNET_malloc (saddrlens[i]);
+#if HAVE_SOCKADDR_IN_SIN_LEN
+      ((struct sockaddr_in *) saddrs[i])->sin_len = saddrlens[i];
+#endif
+      ((struct sockaddr_in *) saddrs[i])->sin_family = AF_INET;
+      ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
+    }
+    else
+    {
+      /* dual stack */
+      resi = 2;
+      if (NULL != unixpath)
+        resi++;
+      saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
+      saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+      i = 0;
+      if (NULL != unixpath)
+      {
+        add_unixpath (saddrs, saddrlens, unixpath, abstract);
+        i++;
+      }
+      saddrlens[i] = sizeof (struct sockaddr_in6);
+      saddrs[i] = GNUNET_malloc (saddrlens[i]);
+#if HAVE_SOCKADDR_IN_SIN_LEN
+      ((struct sockaddr_in6 *) saddrs[i])->sin6_len = saddrlens[0];
+#endif
+      ((struct sockaddr_in6 *) saddrs[i])->sin6_family = AF_INET6;
+      ((struct sockaddr_in6 *) saddrs[i])->sin6_port = htons (port);
+      i++;
+      saddrlens[i] = sizeof (struct sockaddr_in);
+      saddrs[i] = GNUNET_malloc (saddrlens[i]);
+#if HAVE_SOCKADDR_IN_SIN_LEN
+      ((struct sockaddr_in *) saddrs[i])->sin_len = saddrlens[1];
+#endif
+      ((struct sockaddr_in *) saddrs[i])->sin_family = AF_INET;
+      ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
+    }
+  }
+  GNUNET_free_non_null (unixpath);
+  *addrs = saddrs;
+  *addr_lens = saddrlens;
+  return resi;
+}
+
+
+#ifdef MINGW
+/**
+ * Read listen sockets from the parent process (ARM).
+ *
+ * @param sctx service context to initialize
+ * @return #GNUNET_YES if ok, #GNUNET_NO if not ok (must bind yourself),
+ * and #GNUNET_SYSERR on error.
+ */
+static int
+receive_sockets_from_parent (struct LEGACY_SERVICE_Context *sctx)
+{
+  const char *env_buf;
+  int fail;
+  uint64_t count;
+  uint64_t i;
+  HANDLE lsocks_pipe;
+
+  env_buf = getenv ("GNUNET_OS_READ_LSOCKS");
+  if ((NULL == env_buf) || (strlen (env_buf) <= 0))
+    return GNUNET_NO;
+  /* Using W32 API directly here, because this pipe will
+   * never be used outside of this function, and it's just too much of a bother
+   * to create a GNUnet API that boxes a HANDLE (the way it is done with socks)
+   */
+  lsocks_pipe = (HANDLE) strtoul (env_buf, NULL, 10);
+  if ( (0 == lsocks_pipe) || (INVALID_HANDLE_VALUE == lsocks_pipe))
+    return GNUNET_NO;
+  fail = 1;
+  do
+  {
+    int ret;
+    int fail2;
+    DWORD rd;
+
+    ret = ReadFile (lsocks_pipe, &count, sizeof (count), &rd, NULL);
+    if ((0 == ret) || (sizeof (count) != rd) || (0 == count))
+      break;
+    sctx->lsocks =
+        GNUNET_malloc (sizeof (struct GNUNET_NETWORK_Handle *) * (count + 1));
+
+    fail2 = 1;
+    for (i = 0; i < count; i++)
+    {
+      WSAPROTOCOL_INFOA pi;
+      uint64_t size;
+      SOCKET s;
+
+      ret = ReadFile (lsocks_pipe, &size, sizeof (size), &rd, NULL);
+      if ( (0 == ret) || (sizeof (size) != rd) || (sizeof (pi) != size) )
+        break;
+      ret = ReadFile (lsocks_pipe, &pi, sizeof (pi), &rd, NULL);
+      if ( (0 == ret) || (sizeof (pi) != rd))
+        break;
+      s = WSASocketA (pi.iAddressFamily, pi.iSocketType, pi.iProtocol, &pi, 0, WSA_FLAG_OVERLAPPED);
+      sctx->lsocks[i] = GNUNET_NETWORK_socket_box_native (s);
+      if (NULL == sctx->lsocks[i])
+        break;
+      else if (i == count - 1)
+        fail2 = 0;
+    }
+    if (fail2)
+      break;
+    sctx->lsocks[count] = NULL;
+    fail = 0;
+  }
+  while (fail);
+
+  CloseHandle (lsocks_pipe);
+
+  if (fail)
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Could not access a pre-bound socket, will try to bind myself\n"));
+    for (i = 0; (i < count) && (NULL != sctx->lsocks[i]); i++)
+      GNUNET_break (0 == GNUNET_NETWORK_socket_close (sctx->lsocks[i]));
+    GNUNET_free_non_null (sctx->lsocks);
+    sctx->lsocks = NULL;
+    return GNUNET_NO;
+  }
+  return GNUNET_YES;
+}
+#endif
+
+
+/**
+ * Setup addr, addrlen, idle_timeout
+ * based on configuration!
+ *
+ * Configuration may specify:
+ * - PORT (where to bind to for TCP)
+ * - UNIXPATH (where to bind to for UNIX domain sockets)
+ * - TIMEOUT (after how many ms does an inactive service timeout);
+ * - DISABLEV6 (disable support for IPv6, otherwise we use dual-stack)
+ * - BINDTO (hostname or IP address to bind to, otherwise we take everything)
+ * - ACCEPT_FROM  (only allow connections from specified IPv4 subnets)
+ * - ACCEPT_FROM6 (only allow connections from specified IPv6 subnets)
+ * - REJECT_FROM  (disallow allow connections from specified IPv4 subnets)
+ * - REJECT_FROM6 (disallow allow connections from specified IPv6 subnets)
+ *
+ * @param sctx service context to initialize
+ * @return #GNUNET_OK if configuration succeeded
+ */
+static int
+setup_service (struct LEGACY_SERVICE_Context *sctx)
+{
+  struct GNUNET_TIME_Relative idleout;
+  int tolerant;
+
+#ifndef MINGW
+  const char *nfds;
+  unsigned int cnt;
+  int flags;
+#endif
+
+  if (GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, "TIMEOUT"))
+  {
+    if (GNUNET_OK !=
+        GNUNET_CONFIGURATION_get_value_time (sctx->cfg, sctx->service_name,
+                                             "TIMEOUT", &idleout))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Specified value for `%s' of service `%s' is invalid\n"),
+           "TIMEOUT", sctx->service_name);
+      return GNUNET_SYSERR;
+    }
+    sctx->timeout = idleout;
+  }
+  else
+    sctx->timeout = GNUNET_TIME_UNIT_FOREVER_REL;
+
+  if (GNUNET_CONFIGURATION_have_value
+      (sctx->cfg, sctx->service_name, "TOLERANT"))
+  {
+    if (GNUNET_SYSERR ==
+        (tolerant =
+         GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+                                               "TOLERANT")))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Specified value for `%s' of service `%s' is invalid\n"),
+           "TOLERANT", sctx->service_name);
+      return GNUNET_SYSERR;
+    }
+  }
+  else
+    tolerant = GNUNET_NO;
+
+#ifndef MINGW
+  errno = 0;
+  if ((NULL != (nfds = getenv ("LISTEN_FDS"))) &&
+      (1 == SSCANF (nfds, "%u", &cnt)) && (cnt > 0) && (cnt < FD_SETSIZE) &&
+      (cnt + 4 < FD_SETSIZE))
+  {
+    sctx->lsocks =
+        GNUNET_malloc (sizeof (struct GNUNET_NETWORK_Handle *) * (cnt + 1));
+    while (0 < cnt--)
+    {
+      flags = fcntl (3 + cnt, F_GETFD);
+      if ((flags < 0) || (0 != (flags & FD_CLOEXEC)) ||
+          (NULL ==
+           (sctx->lsocks[cnt] = GNUNET_NETWORK_socket_box_native (3 + cnt))))
+      {
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             _
+             ("Could not access pre-bound socket %u, will try to bind myself\n"),
+             (unsigned int) 3 + cnt);
+        cnt++;
+        while (sctx->lsocks[cnt] != NULL)
+          GNUNET_break (0 == GNUNET_NETWORK_socket_close (sctx->lsocks[cnt++]));
+        GNUNET_free (sctx->lsocks);
+        sctx->lsocks = NULL;
+        break;
+      }
+    }
+    unsetenv ("LISTEN_FDS");
+  }
+#else
+  if (getenv ("GNUNET_OS_READ_LSOCKS") != NULL)
+  {
+    receive_sockets_from_parent (sctx);
+    putenv ("GNUNET_OS_READ_LSOCKS=");
+  }
+#endif
+
+  if ((NULL == sctx->lsocks) &&
+      (GNUNET_SYSERR ==
+       LEGACY_SERVICE_get_server_addresses (sctx->service_name, sctx->cfg,
+                                            &sctx->addrs, &sctx->addrlens)))
+    return GNUNET_SYSERR;
+  sctx->require_found = tolerant ? GNUNET_NO : GNUNET_YES;
+  sctx->match_uid =
+      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+                                            "UNIX_MATCH_UID");
+  sctx->match_gid =
+      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+                                            "UNIX_MATCH_GID");
+  process_acl4 (&sctx->v4_denied, sctx, "REJECT_FROM");
+  process_acl4 (&sctx->v4_allowed, sctx, "ACCEPT_FROM");
+  process_acl6 (&sctx->v6_denied, sctx, "REJECT_FROM6");
+  process_acl6 (&sctx->v6_allowed, sctx, "ACCEPT_FROM6");
+
+  return GNUNET_OK;
+}
+
+
+/**
+ * Get the name of the user that'll be used
+ * to provide the service.
+ *
+ * @param sctx service context
+ * @return value of the 'USERNAME' option
+ */
+static char *
+get_user_name (struct LEGACY_SERVICE_Context *sctx)
+{
+  char *un;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->service_name,
+                                               "USERNAME", &un))
+    return NULL;
+  return un;
+}
+
+
+/**
+ * Write PID file.
+ *
+ * @param sctx service context
+ * @param pid PID to write (should be equal to 'getpid()'
+ * @return  #GNUNET_OK on success (including no work to be done)
+ */
+static int
+write_pid_file (struct LEGACY_SERVICE_Context *sctx, pid_t pid)
+{
+  FILE *pidfd;
+  char *pif;
+  char *user;
+  char *rdir;
+  int len;
+
+  if (NULL == (pif = get_pid_file_name (sctx)))
+    return GNUNET_OK;           /* no file desired */
+  user = get_user_name (sctx);
+  rdir = GNUNET_strdup (pif);
+  len = strlen (rdir);
+  while ((len > 0) && (rdir[len] != DIR_SEPARATOR))
+    len--;
+  rdir[len] = '\0';
+  if (0 != ACCESS (rdir, F_OK))
+  {
+    /* we get to create a directory -- and claim it
+     * as ours! */
+    (void) GNUNET_DISK_directory_create (rdir);
+    if ((NULL != user) && (0 < strlen (user)))
+      GNUNET_DISK_file_change_owner (rdir, user);
+  }
+  if (0 != ACCESS (rdir, W_OK | X_OK))
+  {
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_ERROR, "access", rdir);
+    GNUNET_free (rdir);
+    GNUNET_free_non_null (user);
+    GNUNET_free (pif);
+    return GNUNET_SYSERR;
+  }
+  GNUNET_free (rdir);
+  pidfd = FOPEN (pif, "w");
+  if (NULL == pidfd)
+  {
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_ERROR, "fopen", pif);
+    GNUNET_free (pif);
+    GNUNET_free_non_null (user);
+    return GNUNET_SYSERR;
+  }
+  if (0 > FPRINTF (pidfd, "%u", pid))
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "fprintf", pif);
+  GNUNET_break (0 == FCLOSE (pidfd));
+  if ((NULL != user) && (0 < strlen (user)))
+    GNUNET_DISK_file_change_owner (pif, user);
+  GNUNET_free_non_null (user);
+  GNUNET_free (pif);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Task run during shutdown.  Stops the server/service.
+ *
+ * @param cls the `struct LEGACY_SERVICE_Context`
+ */
+static void
+shutdown_task (void *cls)
+{
+  struct LEGACY_SERVICE_Context *service = cls;
+  struct GNUNET_SERVER_Handle *server = service->server;
+
+  service->shutdown_task = NULL;
+  if (0 != (service->options & LEGACY_SERVICE_OPTION_SOFT_SHUTDOWN))
+    GNUNET_SERVER_stop_listening (server);
+  else
+    GNUNET_SERVER_destroy (server);
+}
+
+
+/**
+ * Initial task for the service.
+ *
+ * @param cls service context
+ */
+static void
+service_task (void *cls)
+{
+  struct LEGACY_SERVICE_Context *sctx = cls;
+  unsigned int i;
+
+  GNUNET_RESOLVER_connect (sctx->cfg);
+  if (NULL != sctx->lsocks)
+    sctx->server
+      = GNUNET_SERVER_create_with_sockets (&check_access, sctx, sctx->lsocks,
+                                           sctx->timeout, sctx->require_found);
+  else
+    sctx->server
+      = GNUNET_SERVER_create (&check_access, sctx, sctx->addrs, sctx->addrlens,
+                              sctx->timeout, sctx->require_found);
+  if (NULL == sctx->server)
+  {
+    if (NULL != sctx->addrs)
+      for (i = 0; NULL != sctx->addrs[i]; i++)
+        LOG (GNUNET_ERROR_TYPE_INFO,
+             _("Failed to start `%s' at `%s'\n"),
+             sctx->service_name, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
+    sctx->ret = GNUNET_SYSERR;
+    return;
+  }
+#ifndef WINDOWS
+  if (NULL != sctx->addrs)
+    for (i = 0; NULL != sctx->addrs[i]; i++)
+      if ((AF_UNIX == sctx->addrs[i]->sa_family)
+          && ('\0' != ((const struct sockaddr_un *)sctx->addrs[i])->sun_path[0]))
+        GNUNET_DISK_fix_permissions (((const struct sockaddr_un *)sctx->addrs[i])->sun_path,
+                                     sctx->match_uid,
+                                     sctx->match_gid);
+#endif
+
+
+  if (0 == (sctx->options & LEGACY_SERVICE_OPTION_MANUAL_SHUTDOWN))
+  {
+    /* install a task that will kill the server
+     * process if the scheduler ever gets a shutdown signal */
+    sctx->shutdown_task = GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
+                                                         sctx);
+  }
+  sctx->my_handlers = GNUNET_malloc (sizeof (defhandlers));
+  GNUNET_memcpy (sctx->my_handlers, defhandlers, sizeof (defhandlers));
+  i = 0;
+  while (NULL != sctx->my_handlers[i].callback)
+    sctx->my_handlers[i++].callback_cls = sctx;
+  GNUNET_SERVER_add_handlers (sctx->server, sctx->my_handlers);
+  if (-1 != sctx->ready_confirm_fd)
+  {
+    GNUNET_break (1 == WRITE (sctx->ready_confirm_fd, ".", 1));
+    GNUNET_break (0 == CLOSE (sctx->ready_confirm_fd));
+    sctx->ready_confirm_fd = -1;
+    write_pid_file (sctx, getpid ());
+  }
+  if (NULL != sctx->addrs)
+  {
+    i = 0;
+    while (NULL != sctx->addrs[i])
+    {
+      LOG (GNUNET_ERROR_TYPE_INFO, _("Service `%s' runs at %s\n"),
+           sctx->service_name, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
+      i++;
+    }
+  }
+  sctx->task (sctx->task_cls, sctx->server, sctx->cfg);
+}
+
+
+/**
+ * Detach from terminal.
+ *
+ * @param sctx service context
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+static int
+detach_terminal (struct LEGACY_SERVICE_Context *sctx)
+{
+#ifndef MINGW
+  pid_t pid;
+  int nullfd;
+  int filedes[2];
+
+  if (0 != PIPE (filedes))
+  {
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "pipe");
+    return GNUNET_SYSERR;
+  }
+  pid = fork ();
+  if (pid < 0)
+  {
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "fork");
+    return GNUNET_SYSERR;
+  }
+  if (0 != pid)
+  {
+    /* Parent */
+    char c;
+
+    GNUNET_break (0 == CLOSE (filedes[1]));
+    c = 'X';
+    if (1 != READ (filedes[0], &c, sizeof (char)))
+      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "read");
+    fflush (stdout);
+    switch (c)
+    {
+    case '.':
+      exit (0);
+    case 'I':
+      LOG (GNUNET_ERROR_TYPE_INFO, _("Service process failed to initialize\n"));
+      break;
+    case 'S':
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Service process could not initialize server function\n"));
+      break;
+    case 'X':
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Service process failed to report status\n"));
+      break;
+    }
+    exit (1);                   /* child reported error */
+  }
+  GNUNET_break (0 == CLOSE (0));
+  GNUNET_break (0 == CLOSE (1));
+  GNUNET_break (0 == CLOSE (filedes[0]));
+  nullfd = OPEN ("/dev/null", O_RDWR | O_APPEND);
+  if (nullfd < 0)
+    return GNUNET_SYSERR;
+  /* set stdin/stdout to /dev/null */
+  if ((dup2 (nullfd, 0) < 0) || (dup2 (nullfd, 1) < 0))
+  {
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "dup2");
+    (void) CLOSE (nullfd);
+    return GNUNET_SYSERR;
+  }
+  (void) CLOSE (nullfd);
+  /* Detach from controlling terminal */
+  pid = setsid ();
+  if (-1 == pid)
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "setsid");
+  sctx->ready_confirm_fd = filedes[1];
+#else
+  /* FIXME: we probably need to do something else
+   * elsewhere in order to fork the process itself... */
+  FreeConsole ();
+#endif
+  return GNUNET_OK;
+}
+
+
+/**
+ * Set user ID.
+ *
+ * @param sctx service context
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ */
+static int
+set_user_id (struct LEGACY_SERVICE_Context *sctx)
+{
+  char *user;
+
+  if (NULL == (user = get_user_name (sctx)))
+    return GNUNET_OK;           /* keep */
+#ifndef MINGW
+  struct passwd *pws;
+
+  errno = 0;
+  pws = getpwnam (user);
+  if (NULL == pws)
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Cannot obtain information about user `%s': %s\n"), user,
+         errno == 0 ? _("No such user") : STRERROR (errno));
+    GNUNET_free (user);
+    return GNUNET_SYSERR;
+  }
+  if ((0 != setgid (pws->pw_gid)) || (0 != setegid (pws->pw_gid)) ||
+#if HAVE_INITGROUPS
+      (0 != initgroups (user, pws->pw_gid)) ||
+#endif
+      (0 != setuid (pws->pw_uid)) || (0 != seteuid (pws->pw_uid)))
+  {
+    if ((0 != setregid (pws->pw_gid, pws->pw_gid)) ||
+        (0 != setreuid (pws->pw_uid, pws->pw_uid)))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR, _("Cannot change user/group to `%s': %s\n"),
+           user, STRERROR (errno));
+      GNUNET_free (user);
+      return GNUNET_SYSERR;
+    }
+  }
+#endif
+  GNUNET_free (user);
+  return GNUNET_OK;
+}
+
+
+/**
+ * Delete the PID file that was created by our parent.
+ *
+ * @param sctx service context
+ */
+static void
+pid_file_delete (struct LEGACY_SERVICE_Context *sctx)
+{
+  char *pif = get_pid_file_name (sctx);
+
+  if (NULL == pif)
+    return;                     /* no PID file */
+  if (0 != UNLINK (pif))
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "unlink", pif);
+  GNUNET_free (pif);
+}
+
+
+/**
+ * Run a standard GNUnet service startup sequence (initialize loggers
+ * and configuration, parse options).
+ *
+ * @param argc number of command line arguments
+ * @param argv command line arguments
+ * @param service_name our service name
+ * @param options service options
+ * @param task main task of the service
+ * @param task_cls closure for @a task
+ * @return #GNUNET_SYSERR on error, #GNUNET_OK
+ *         if we shutdown nicely
+ */
+int
+LEGACY_SERVICE_run (int argc, char *const *argv,
+                    const char *service_name,
+                    enum LEGACY_SERVICE_Options options,
+                    LEGACY_SERVICE_Main task,
+                    void *task_cls)
+{
+#define HANDLE_ERROR do { GNUNET_break (0); goto shutdown; } while (0)
+
+  int err;
+  int ret;
+  char *cfg_fn;
+  char *opt_cfg_fn;
+  char *loglev;
+  char *logfile;
+  int do_daemonize;
+  unsigned int i;
+  unsigned long long skew_offset;
+  unsigned long long skew_variance;
+  long long clock_offset;
+  struct LEGACY_SERVICE_Context sctx;
+  struct GNUNET_CONFIGURATION_Handle *cfg;
+  const char *xdg;
+
+  struct GNUNET_GETOPT_CommandLineOption service_options[] = {
+    GNUNET_GETOPT_OPTION_CFG_FILE (&opt_cfg_fn),
+    GNUNET_GETOPT_OPTION_SET_ONE ('d',
+                                  "daemonize",
+                                  gettext_noop ("do daemonize (detach from terminal)"),
+                                  &do_daemonize),
+    GNUNET_GETOPT_OPTION_HELP (NULL),
+    GNUNET_GETOPT_OPTION_LOGLEVEL (&loglev),
+    GNUNET_GETOPT_OPTION_LOGFILE (&logfile),
+    GNUNET_GETOPT_OPTION_VERSION (PACKAGE_VERSION " " VCS_VERSION),
+    GNUNET_GETOPT_OPTION_END
+  };
+  err = 1;
+  do_daemonize = 0;
+  logfile = NULL;
+  loglev = NULL;
+  opt_cfg_fn = NULL;
+  xdg = getenv ("XDG_CONFIG_HOME");
+  if (NULL != xdg)
+    GNUNET_asprintf (&cfg_fn,
+                     "%s%s%s",
+                     xdg,
+                     DIR_SEPARATOR_STR,
+                     GNUNET_OS_project_data_get ()->config_file);
+  else
+    cfg_fn = GNUNET_strdup (GNUNET_OS_project_data_get ()->user_config_file);
+  memset (&sctx, 0, sizeof (sctx));
+  sctx.options = options;
+  sctx.ready_confirm_fd = -1;
+  sctx.ret = GNUNET_OK;
+  sctx.timeout = GNUNET_TIME_UNIT_FOREVER_REL;
+  sctx.task = task;
+  sctx.task_cls = task_cls;
+  sctx.service_name = service_name;
+  sctx.cfg = cfg = GNUNET_CONFIGURATION_create ();
+
+  /* setup subsystems */
+  ret = GNUNET_GETOPT_run (service_name, service_options, argc, argv);
+  if (GNUNET_SYSERR == ret)
+    goto shutdown;
+  if (GNUNET_NO == ret)
+  {
+    err = 0;
+    goto shutdown;
+  }
+  if (GNUNET_OK != GNUNET_log_setup (service_name, loglev, logfile))
+    HANDLE_ERROR;
+  if (NULL == opt_cfg_fn)
+    opt_cfg_fn = GNUNET_strdup (cfg_fn);
+  if (GNUNET_YES == GNUNET_DISK_file_test (opt_cfg_fn))
+  {
+    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg, opt_cfg_fn))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Malformed configuration file `%s', exit ...\n"),
+                  opt_cfg_fn);
+      goto shutdown;
+    }
+  }
+  else
+  {
+    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg, NULL))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Malformed configuration, exit ...\n"));
+      goto shutdown;
+    }
+    if (0 != strcmp (opt_cfg_fn, cfg_fn))
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Could not access configuration file `%s'\n"),
+                  opt_cfg_fn);
+  }
+  if (GNUNET_OK != setup_service (&sctx))
+    goto shutdown;
+  if ((1 == do_daemonize) && (GNUNET_OK != detach_terminal (&sctx)))
+    HANDLE_ERROR;
+  if (GNUNET_OK != set_user_id (&sctx))
+    goto shutdown;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Service `%s' runs with configuration from `%s'\n",
+       service_name,
+       opt_cfg_fn);
+  if ((GNUNET_OK ==
+       GNUNET_CONFIGURATION_get_value_number (sctx.cfg, "TESTING",
+                                              "SKEW_OFFSET", &skew_offset)) &&
+      (GNUNET_OK ==
+       GNUNET_CONFIGURATION_get_value_number (sctx.cfg, "TESTING",
+                                              "SKEW_VARIANCE", &skew_variance)))
+  {
+    clock_offset = skew_offset - skew_variance;
+    GNUNET_TIME_set_offset (clock_offset);
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "Skewing clock by %dll ms\n", clock_offset);
+  }
+  /* actually run service */
+  err = 0;
+  GNUNET_SCHEDULER_run (&service_task, &sctx);
+  /* shutdown */
+  if ((1 == do_daemonize) && (NULL != sctx.server))
+    pid_file_delete (&sctx);
+  GNUNET_free_non_null (sctx.my_handlers);
+
+shutdown:
+  if (-1 != sctx.ready_confirm_fd)
+  {
+    if (1 != WRITE (sctx.ready_confirm_fd, err ? "I" : "S", 1))
+      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "write");
+    GNUNET_break (0 == CLOSE (sctx.ready_confirm_fd));
+  }
+#if HAVE_MALLINFO
+  {
+    char *counter;
+
+    if ( (GNUNET_YES ==
+          GNUNET_CONFIGURATION_have_value (sctx.cfg, service_name,
+                                           "GAUGER_HEAP")) &&
+         (GNUNET_OK ==
+          GNUNET_CONFIGURATION_get_value_string (sctx.cfg, service_name,
+                                                 "GAUGER_HEAP",
+                                                 &counter)) )
+    {
+      struct mallinfo mi;
+
+      mi = mallinfo ();
+      GAUGER (service_name, counter, mi.usmblks, "blocks");
+      GNUNET_free (counter);
+    }
+  }
+#endif
+  GNUNET_CONFIGURATION_destroy (cfg);
+  i = 0;
+  if (NULL != sctx.addrs)
+    while (NULL != sctx.addrs[i])
+      GNUNET_free (sctx.addrs[i++]);
+  GNUNET_free_non_null (sctx.addrs);
+  GNUNET_free_non_null (sctx.addrlens);
+  GNUNET_free_non_null (logfile);
+  GNUNET_free_non_null (loglev);
+  GNUNET_free (cfg_fn);
+  GNUNET_free_non_null (opt_cfg_fn);
+  GNUNET_free_non_null (sctx.v4_denied);
+  GNUNET_free_non_null (sctx.v6_denied);
+  GNUNET_free_non_null (sctx.v4_allowed);
+  GNUNET_free_non_null (sctx.v6_allowed);
+
+  return err ? GNUNET_SYSERR : sctx.ret;
+}
+
+
+/**
+ * Run a service startup sequence within an existing
+ * initialized system.
+ *
+ * @param service_name our service name
+ * @param cfg configuration to use
+ * @param options service options
+ * @return NULL on error, service handle
+ */
+struct LEGACY_SERVICE_Context *
+LEGACY_SERVICE_start (const char *service_name,
+                      const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      enum LEGACY_SERVICE_Options options)
+{
+  int i;
+  struct LEGACY_SERVICE_Context *sctx;
+
+  sctx = GNUNET_new (struct LEGACY_SERVICE_Context);
+  sctx->ready_confirm_fd = -1;  /* no daemonizing */
+  sctx->ret = GNUNET_OK;
+  sctx->timeout = GNUNET_TIME_UNIT_FOREVER_REL;
+  sctx->service_name = service_name;
+  sctx->cfg = cfg;
+  sctx->options = options;
+
+  /* setup subsystems */
+  if (GNUNET_OK != setup_service (sctx))
+  {
+    LEGACY_SERVICE_stop (sctx);
+    return NULL;
+  }
+  if (NULL != sctx->lsocks)
+    sctx->server =
+        GNUNET_SERVER_create_with_sockets (&check_access, sctx, sctx->lsocks,
+                                           sctx->timeout, sctx->require_found);
+  else
+    sctx->server =
+        GNUNET_SERVER_create (&check_access, sctx, sctx->addrs, sctx->addrlens,
+                              sctx->timeout, sctx->require_found);
+
+  if (NULL == sctx->server)
+  {
+    LEGACY_SERVICE_stop (sctx);
+    return NULL;
+  }
+#ifndef WINDOWS
+  if (NULL != sctx->addrs)
+    for (i = 0; NULL != sctx->addrs[i]; i++)
+      if ((AF_UNIX == sctx->addrs[i]->sa_family)
+          && ('\0' != ((const struct sockaddr_un *)sctx->addrs[i])->sun_path[0]))
+        GNUNET_DISK_fix_permissions (((const struct sockaddr_un *)sctx->addrs[i])->sun_path,
+                                     sctx->match_uid,
+                                     sctx->match_gid);
+#endif
+  sctx->my_handlers = GNUNET_malloc (sizeof (defhandlers));
+  GNUNET_memcpy (sctx->my_handlers, defhandlers, sizeof (defhandlers));
+  i = 0;
+  while ((sctx->my_handlers[i].callback != NULL))
+    sctx->my_handlers[i++].callback_cls = sctx;
+  GNUNET_SERVER_add_handlers (sctx->server, sctx->my_handlers);
+  return sctx;
+}
+
+
+/**
+ * Obtain the server used by a service.  Note that the server must NOT
+ * be destroyed by the caller.
+ *
+ * @param ctx the service context returned from the start function
+ * @return handle to the server for this service, NULL if there is none
+ */
+struct GNUNET_SERVER_Handle *
+LEGACY_SERVICE_get_server (struct LEGACY_SERVICE_Context *ctx)
+{
+  return ctx->server;
+}
+
+
+/**
+ * Get the NULL-terminated array of listen sockets for this service.
+ *
+ * @param ctx service context to query
+ * @return NULL if there are no listen sockets, otherwise NULL-terminated
+ *              array of listen sockets.
+ */
+struct GNUNET_NETWORK_Handle *const*
+LEGACY_SERVICE_get_listen_sockets (struct LEGACY_SERVICE_Context *ctx)
+{
+  return ctx->lsocks;
+}
+
+
+/**
+ * Stop a service that was started with "LEGACY_SERVICE_start".
+ *
+ * @param sctx the service context returned from the start function
+ */
+void
+LEGACY_SERVICE_stop (struct LEGACY_SERVICE_Context *sctx)
+{
+  unsigned int i;
+
+#if HAVE_MALLINFO
+  {
+    char *counter;
+
+    if ( (GNUNET_YES ==
+          GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name,
+                                           "GAUGER_HEAP")) &&
+         (GNUNET_OK ==
+          GNUNET_CONFIGURATION_get_value_string (sctx->cfg, sctx->service_name,
+                                                 "GAUGER_HEAP",
+                                                 &counter)) )
+    {
+      struct mallinfo mi;
+
+      mi = mallinfo ();
+      GAUGER (sctx->service_name, counter, mi.usmblks, "blocks");
+      GNUNET_free (counter);
+    }
+  }
+#endif
+  if (NULL != sctx->shutdown_task)
+  {
+    GNUNET_SCHEDULER_cancel (sctx->shutdown_task);
+    sctx->shutdown_task = NULL;
+  }
+  if (NULL != sctx->server)
+    GNUNET_SERVER_destroy (sctx->server);
+  GNUNET_free_non_null (sctx->my_handlers);
+  if (NULL != sctx->addrs)
+  {
+    i = 0;
+    while (NULL != sctx->addrs[i])
+      GNUNET_free (sctx->addrs[i++]);
+    GNUNET_free (sctx->addrs);
+  }
+  GNUNET_free_non_null (sctx->addrlens);
+  GNUNET_free_non_null (sctx->v4_denied);
+  GNUNET_free_non_null (sctx->v6_denied);
+  GNUNET_free_non_null (sctx->v4_allowed);
+  GNUNET_free_non_null (sctx->v6_allowed);
+  GNUNET_free (sctx);
+}
+
+
+/* end of service.c */
diff --git a/src/transport/test_plugin_transport.c b/src/transport/test_plugin_transport.c
index be79d5499..1d92588ea 100644
--- a/src/transport/test_plugin_transport.c
+++ b/src/transport/test_plugin_transport.c
@@ -552,7 +552,7 @@ setup_plugin_environment ()
 
 
 static int
-handle_helper_message (void *cls, void *client,
+handle_helper_message (void *cls,
                        const struct GNUNET_MessageHeader *hdr)
 {
   return GNUNET_OK;
diff --git a/src/transport/test_transport_api_reliability.c b/src/transport/test_transport_api_reliability.c
index 375a935c8..e5ba2831b 100644
--- a/src/transport/test_transport_api_reliability.c
+++ b/src/transport/test_transport_api_reliability.c
@@ -217,7 +217,7 @@ notify_receive (void *cls,
 {
   static int n;
   unsigned int s;
-  char cbuf[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
+  char cbuf[GNUNET_MAX_MESSAGE_SIZE - 1];
 
   if (GNUNET_TRANSPORT_TESTING_SIMPLE_MTYPE != ntohs (hdr->header.type))
     return;
diff --git a/src/transport/transport-testing-loggers.c b/src/transport/transport-testing-loggers.c
index de9fa91c1..0ebb07d74 100644
--- a/src/transport/transport-testing-loggers.c
+++ b/src/transport/transport-testing-loggers.c
@@ -71,6 +71,7 @@ GNUNET_TRANSPORT_TESTING_log_disconnect (void *cls,
               GNUNET_i2s (other),
               me->no,
               ps);
+  GNUNET_free (ps);
 }
 
 /* end of transport-testing-loggers.c */
diff --git a/src/transport/transport-testing-main.c b/src/transport/transport-testing-main.c
index a79d81cb9..81a66e113 100644
--- a/src/transport/transport-testing-main.c
+++ b/src/transport/transport-testing-main.c
@@ -146,7 +146,7 @@ struct GNUNET_TRANSPORT_TESTING_InternalPeerContext
 
 /**
  * Information tracked per connected peer.
- */ 
+ */
 struct ConnectPairInfo
 {
   /**
@@ -235,7 +235,7 @@ my_nc (void *cls,
   struct GNUNET_TRANSPORT_TESTING_InternalPeerContext *ipi = cls;
   struct GNUNET_TRANSPORT_TESTING_ConnectCheckContext *ccc = ipi->ccc;
   struct ConnectPairInfo *cpi;
-  
+
   if (NULL != ccc->nc)
     ccc->nc (ccc->cls,
              ccc->p[ipi->off],
@@ -262,7 +262,7 @@ my_nd (void *cls,
   struct GNUNET_TRANSPORT_TESTING_InternalPeerContext *ipi = cls;
   struct GNUNET_TRANSPORT_TESTING_ConnectCheckContext *ccc = ipi->ccc;
   struct ConnectPairInfo *cpi = custom_cls;
-  
+
   if (NULL != ccc->nd)
     ccc->nd (ccc->cls,
              ccc->p[ipi->off],
@@ -535,13 +535,15 @@ GNUNET_TRANSPORT_TESTING_connect_check (void *cls,
     ip[i].off = i;
     ip[i].ccc = ccc;
   }
-  GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
-                      argv,
-                      test_name_,
-                      "nohelp",
-                      options,
-                      &connect_check_run,
-                      ccc);
+  if (GNUNET_OK !=
+      GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
+                          argv,
+                          test_name_,
+                          "nohelp",
+                          options,
+                          &connect_check_run,
+                          ccc))
+    return GNUNET_SYSERR;
   return ccc->global_ret;
 }
 
diff --git a/src/transport/transport-testing.c b/src/transport/transport-testing.c
index c0775a135..53a44f338 100644
--- a/src/transport/transport-testing.c
+++ b/src/transport/transport-testing.c
@@ -49,7 +49,7 @@ find_peer_context (struct GNUNET_TRANSPORT_TESTING_Handle *tth,
  *
  * @param p1 first peer
  * @param p2 second peer
- * @param cb function to call 
+ * @param cb function to call
  * @param cb_cls closure for @a cb
  */
 void
@@ -66,7 +66,7 @@ GNUNET_TRANSPORT_TESTING_find_connecting_context (struct GNUNET_TRANSPORT_TESTIN
   {
     ccn = cc->next;
     if ( (cc->p1 == p1) &&
-         (cc->p2 == p2) ) 
+         (cc->p2 == p2) )
       cb (cb_cls,
           cc);
   }
@@ -74,7 +74,7 @@ GNUNET_TRANSPORT_TESTING_find_connecting_context (struct GNUNET_TRANSPORT_TESTIN
 
 
 static void
-set_p1c (void *cls,   
+set_p1c (void *cls,
          struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cx)
 {
   int *found = cls;
@@ -86,7 +86,7 @@ set_p1c (void *cls,
 
 
 static void
-set_mq (void *cls,   
+set_mq (void *cls,
         struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cx)
 {
   struct GNUNET_MQ_Handle *mq = cls;
@@ -96,7 +96,7 @@ set_mq (void *cls,
 
 
 static void
-set_p2c (void *cls,   
+set_p2c (void *cls,
          struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cx)
 {
   int *found = cls;
@@ -108,7 +108,7 @@ set_p2c (void *cls,
 
 
 static void
-clear_p1c (void *cls,   
+clear_p1c (void *cls,
            struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cx)
 {
   int *found = cls;
@@ -120,7 +120,7 @@ clear_p1c (void *cls,
 
 
 static void
-clear_p2c (void *cls,   
+clear_p2c (void *cls,
          struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cx)
 {
   int *found = cls;
@@ -154,7 +154,7 @@ notify_connect (void *cls,
   else
     ret = NULL;
 
-  if (p2 != NULL)
+  if (NULL != p2)
     GNUNET_asprintf (&p2_s,
                      "%u (`%s')",
                      p2->no,
@@ -267,7 +267,7 @@ notify_disconnect (void *cls,
   int no = 0;
   struct GNUNET_TRANSPORT_TESTING_PeerContext *p2 = NULL;
   struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cc;
-  
+
   p2 = find_peer_context (p->tth,
                           peer);
   no = p->no;
@@ -386,7 +386,7 @@ GNUNET_TRANSPORT_TESTING_start_peer (struct GNUNET_TRANSPORT_TESTING_Handle *tth
   struct GNUNET_TRANSPORT_TESTING_PeerContext *p;
   struct GNUNET_PeerIdentity *dummy;
   unsigned int i;
-  
+
   if (GNUNET_NO == GNUNET_DISK_file_test (cfgname))
   {
     LOG (GNUNET_ERROR_TYPE_ERROR,
@@ -407,7 +407,7 @@ GNUNET_TRANSPORT_TESTING_start_peer (struct GNUNET_TRANSPORT_TESTING_Handle *tth
     GNUNET_memcpy (p->handlers,
                    handlers,
                    i * sizeof (struct GNUNET_MQ_MessageHandler));
-  }    
+  }
   if (NULL != cb_cls)
     p->cb_cls = cb_cls;
   else
@@ -532,7 +532,7 @@ GNUNET_TRANSPORT_TESTING_restart_peer (struct GNUNET_TRANSPORT_TESTING_PeerConte
 {
   struct GNUNET_TRANSPORT_TESTING_ConnectRequest *cc;
   struct GNUNET_TRANSPORT_TESTING_ConnectRequest *ccn;
- 
+
   /* shutdown */
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Stopping peer %u (`%s')\n",
@@ -770,7 +770,7 @@ GNUNET_TRANSPORT_TESTING_connect_peers (struct GNUNET_TRANSPORT_TESTING_PeerCont
       break;
     }
   }
-          
+
   cc = GNUNET_new (struct GNUNET_TRANSPORT_TESTING_ConnectRequest);
   cc->p1 = p1;
   cc->p2 = p2;
diff --git a/src/transport/transport_api_address_to_string.c b/src/transport/transport_api_address_to_string.c
index b9c72dcb3..902764a8f 100644
--- a/src/transport/transport_api_address_to_string.c
+++ b/src/transport/transport_api_address_to_string.c
@@ -199,10 +199,10 @@ GNUNET_TRANSPORT_address_to_string (const struct GNUNET_CONFIGURATION_Handle *cf
 
   alen = address->address_length;
   slen = strlen (address->transport_name) + 1;
-  if ( (alen + slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE
+  if ( (alen + slen >= GNUNET_MAX_MESSAGE_SIZE
         - sizeof (struct AddressLookupMessage)) ||
-       (alen >= GNUNET_SERVER_MAX_MESSAGE_SIZE) ||
-       (slen >= GNUNET_SERVER_MAX_MESSAGE_SIZE) )
+       (alen >= GNUNET_MAX_MESSAGE_SIZE) ||
+       (slen >= GNUNET_MAX_MESSAGE_SIZE) )
   {
     GNUNET_break (0);
     GNUNET_free (alc);
diff --git a/src/transport/transport_api_core.c b/src/transport/transport_api_core.c
index c99ade92f..9c29d4908 100644
--- a/src/transport/transport_api_core.c
+++ b/src/transport/transport_api_core.c
@@ -425,7 +425,7 @@ mq_send_impl (struct GNUNET_MQ_Handle *mq,
 
   GNUNET_assert (GNUNET_YES == n->is_ready);
   msize = ntohs (msg->size);
-  if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (*obm))
+  if (msize >= GNUNET_MAX_MESSAGE_SIZE - sizeof (*obm))
   {
     GNUNET_break (0);
     GNUNET_MQ_impl_send_continue (mq);
diff --git a/src/util/Makefile.am b/src/util/Makefile.am
index df319fe77..9be572bb6 100644
--- a/src/util/Makefile.am
+++ b/src/util/Makefile.am
@@ -30,10 +30,9 @@ W32CONSOLEHELPER = gnunet-helper-w32-console
 endif
 
 if !MINGW
- SERVER_CLIENT_UNIX = test_server_with_client_unix
- TEST_CLIENT_UNIC_NC = test_client_unix.nc
+ TEST_CLIENT_UNIX_NC = test_client_unix.nc
 else
- TEST_CLIENT_UNIC_NC =
+ TEST_CLIENT_UNIX_NC =
 endif
 
 if USE_COVERAGE
@@ -68,7 +67,6 @@ libgnunetutil_la_SOURCES = \
   common_logging.c \
   configuration.c \
   configuration_loader.c \
-  connection.c \
   container_bloomfilter.c \
   container_heap.c \
   container_meta_data.c \
@@ -108,16 +106,10 @@ libgnunetutil_la_SOURCES = \
   program.c \
   resolver_api.c resolver.h \
   scheduler.c \
-  server.c \
-  server_mst.c \
-  server_nc.c \
-  server_tc.c \
   service.c \
-  service_new.c \
   signal.c \
   strings.c \
   time.c \
-  socks.c \
   speedup.c speedup.h
 
 libgnunetutil_la_LIBADD = \
@@ -263,14 +255,13 @@ if HAVE_BENCHMARKS
 endif
 
 if HAVE_SSH_KEY
- SSH_USING_TESTS = test_socks.nc
+# SSH_USING_TESTS = test_socks.nc
 endif
 
 check_PROGRAMS = \
  test_bio \
  test_client.nc \
  $(TEST_CLIENT_UNIX_NC) \
- $(SSH_USING_TESTS) \
  test_common_allocation \
  test_common_endian \
  test_common_logging \
@@ -298,12 +289,6 @@ check_PROGRAMS = \
  test_crypto_rsa \
  test_disk \
  test_getopt \
- test_connection.nc \
- test_connection_addressing.nc \
- test_connection_receive_cancel.nc \
- test_connection_timeout.nc \
- test_connection_timeout_no_connect.nc \
- test_connection_transmit_cancel.nc \
  test_mq \
  test_os_network \
  test_peer \
@@ -312,11 +297,6 @@ check_PROGRAMS = \
  test_resolver_api.nc \
  test_scheduler \
  test_scheduler_delay \
- test_server.nc \
- test_server_disconnect.nc \
- test_server_with_client.nc \
- test_server_mst_interrupt.nc \
- $(SERVER_CLIENT_UNIX) \
  test_service \
  test_strings \
  test_strings_to_data \
@@ -330,18 +310,7 @@ check_PROGRAMS = \
 # Declare .nc (NO-CONCURRENCY) as a test extension so that we can impart
 # sequential execution order for them
 TEST_EXTENSIONS = .nc
-test_connection.log: test_client.log
-test_connection_addressing.log: test_connection.log
-test_connection_timeout_no_connect.log: test_connection_addressing.log
-test_connection_transmit_cancel.log: test_connection_timeout_no_connect.log
-test_connection_receive_cancel.log: test_connection_transmit_cancel.log
-test_connection_timeout.log: test_connection_receive_cancel.log
-test_resolver_api.log: test_connection_timeout.log
-test_server.log: test_resolver_api.log
-test_server_disconnect.log: test_server.log
-test_server_with_client.log: test_server_disconnect.log
-test_server_mst_interrupt.log: test_server_with_client.log
-test_client_unix.log: test_server_mst_interrupt.log
+test_test_client_unix.log: test_client.log
 
 test_bio_SOURCES = \
  test_bio.c
@@ -518,36 +487,6 @@ test_getopt_SOURCES = \
 test_getopt_LDADD = \
  libgnunetutil.la
 
-test_connection_nc_SOURCES = \
- test_connection.c
-test_connection_nc_LDADD = \
- libgnunetutil.la
-
-test_connection_addressing_nc_SOURCES = \
- test_connection_addressing.c
-test_connection_addressing_nc_LDADD = \
- libgnunetutil.la
-
-test_connection_receive_cancel_nc_SOURCES = \
- test_connection_receive_cancel.c
-test_connection_receive_cancel_nc_LDADD = \
- libgnunetutil.la
-
-test_connection_timeout_nc_SOURCES = \
- test_connection_timeout.c
-test_connection_timeout_nc_LDADD = \
- libgnunetutil.la
-
-test_connection_timeout_no_connect_nc_SOURCES = \
- test_connection_timeout_no_connect.c
-test_connection_timeout_no_connect_nc_LDADD = \
- libgnunetutil.la
-
-test_connection_transmit_cancel_nc_SOURCES = \
- test_connection_transmit_cancel.c
-test_connection_transmit_cancel_nc_LDADD = \
- libgnunetutil.la
-
 test_mq_SOURCES = \
  test_mq.c
 test_mq_LDADD = \
@@ -588,32 +527,6 @@ test_scheduler_delay_SOURCES = \
 test_scheduler_delay_LDADD = \
  libgnunetutil.la
 
-test_server_mst_interrupt_nc_SOURCES = \
- test_server_mst_interrupt.c
-test_server_mst_interrupt_nc_LDADD = \
- libgnunetutil.la
-
-test_server_nc_SOURCES = \
- test_server.c
-test_server_nc_LDADD = \
- libgnunetutil.la
-
-test_server_disconnect_nc_SOURCES = \
- test_server_disconnect.c
-test_server_disconnect_nc_LDADD = \
- libgnunetutil.la
-
-test_server_with_client_nc_SOURCES = \
- test_server_with_client.c
-test_server_with_client_nc_LDADD = \
- libgnunetutil.la
-
-test_server_with_client_unix_SOURCES = \
- test_server_with_client_unix.c
-test_server_with_client_unix_LDADD = \
- libgnunetutil.la
-
-
 test_service_SOURCES = \
  test_service.c
 test_service_LDADD = \
@@ -624,7 +537,6 @@ test_strings_SOURCES = \
 test_strings_LDADD = \
  libgnunetutil.la
 
-
 test_strings_to_data_SOURCES = \
  test_strings_to_data.c
 test_strings_to_data_LDADD = \
diff --git a/src/util/bandwidth.c b/src/util/bandwidth.c
index a059fc738..bc0c3b9b4 100644
--- a/src/util/bandwidth.c
+++ b/src/util/bandwidth.c
@@ -184,8 +184,8 @@ update_excess (struct GNUNET_BANDWIDTH_Tracker *av)
   }
   /* negative current_consumption means that we have savings */
   max_carry = ((uint64_t) av->available_bytes_per_s__) * av->max_carry_s__;
-  if (max_carry < GNUNET_SERVER_MAX_MESSAGE_SIZE)
-    max_carry = GNUNET_SERVER_MAX_MESSAGE_SIZE;
+  if (max_carry < GNUNET_MAX_MESSAGE_SIZE)
+    max_carry = GNUNET_MAX_MESSAGE_SIZE;
   if (max_carry > INT64_MAX)
     max_carry = INT64_MAX;
   left_bytes = current_consumption + max_carry;
@@ -224,10 +224,10 @@ update_excess (struct GNUNET_BANDWIDTH_Tracker *av)
 /**
  * Initialize bandwidth tracker.  Note that in addition to the
  * 'max_carry_s' limit, we also always allow at least
- * #GNUNET_SERVER_MAX_MESSAGE_SIZE to accumulate.  So if the
+ * #GNUNET_MAX_MESSAGE_SIZE to accumulate.  So if the
  * bytes-per-second limit is so small that within 'max_carry_s' not
- * even #GNUNET_SERVER_MAX_MESSAGE_SIZE is allowed to accumulate, it is
- * ignored and replaced by #GNUNET_SERVER_MAX_MESSAGE_SIZE (which is in
+ * even #GNUNET_MAX_MESSAGE_SIZE is allowed to accumulate, it is
+ * ignored and replaced by #GNUNET_MAX_MESSAGE_SIZE (which is in
  * bytes).
  *
  * To stop notifications about updates and excess callbacks use
@@ -271,10 +271,10 @@ GNUNET_BANDWIDTH_tracker_init2 (struct GNUNET_BANDWIDTH_Tracker *av,
 /**
  * Initialize bandwidth tracker.  Note that in addition to the
  * 'max_carry_s' limit, we also always allow at least
- * #GNUNET_SERVER_MAX_MESSAGE_SIZE to accumulate.  So if the
+ * #GNUNET_MAX_MESSAGE_SIZE to accumulate.  So if the
  * bytes-per-second limit is so small that within 'max_carry_s' not
- * even #GNUNET_SERVER_MAX_MESSAGE_SIZE is allowed to accumulate, it is
- * ignored and replaced by #GNUNET_SERVER_MAX_MESSAGE_SIZE (which is in
+ * even #GNUNET_MAX_MESSAGE_SIZE is allowed to accumulate, it is
+ * ignored and replaced by #GNUNET_MAX_MESSAGE_SIZE (which is in
  * bytes).
  *
  * @param av tracker to initialize
@@ -345,8 +345,8 @@ update_tracker (struct GNUNET_BANDWIDTH_Tracker *av)
     left_bytes = - av->consumption_since_last_update__;
     max_carry = ((unsigned long long) av->available_bytes_per_s__) *
                 av->max_carry_s__;
-    if (max_carry < GNUNET_SERVER_MAX_MESSAGE_SIZE)
-      max_carry = GNUNET_SERVER_MAX_MESSAGE_SIZE;
+    if (max_carry < GNUNET_MAX_MESSAGE_SIZE)
+      max_carry = GNUNET_MAX_MESSAGE_SIZE;
     if (max_carry > INT64_MAX)
       max_carry = INT64_MAX;
     if (max_carry > left_bytes)
diff --git a/src/util/client.c b/src/util/client.c
index 0f7d0d359..3d74bff33 100644
--- a/src/util/client.c
+++ b/src/util/client.c
@@ -35,6 +35,15 @@
 
 #define LOG(kind,...) GNUNET_log_from (kind, "util-client",__VA_ARGS__)
 
+/**
+ * Timeout we use on TCP connect before trying another
+ * result from the DNS resolver.  Actual value used
+ * is this value divided by the number of address families.
+ * Default is 5s.
+ */
+#define CONNECT_RETRY_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
+
+
 
 /**
  * Internal state for a client connected to a GNUnet service.
@@ -298,11 +307,11 @@ recv_message (void *cls,
 
   if (GNUNET_YES == cstate->in_destroy)
     return GNUNET_SYSERR;
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Received message of type %u and size %u from %s\n",
-       ntohs (msg->type), ntohs (msg->size), cstate->service_name);
-
+       ntohs (msg->type),
+       ntohs (msg->size),
+       cstate->service_name);
   GNUNET_MQ_inject_message (cstate->mq,
                             msg);
   if (GNUNET_YES == cstate->in_destroy)
@@ -656,7 +665,7 @@ try_connect_using_address (void *cls,
   GNUNET_CONTAINER_DLL_insert (cstate->ap_head,
                                cstate->ap_tail,
                                ap);
-  ap->task = GNUNET_SCHEDULER_add_write_net (GNUNET_CONNECTION_CONNECT_RETRY_TIMEOUT,
+  ap->task = GNUNET_SCHEDULER_add_write_net (CONNECT_RETRY_TIMEOUT,
                                              ap->sock,
                                              &connect_probe_continuation,
                                              ap);
@@ -760,7 +769,7 @@ start_connect (void *cls)
   cstate->dns_active
     = GNUNET_RESOLVER_ip_get (cstate->hostname,
                               AF_UNSPEC,
-                              GNUNET_CONNECTION_CONNECT_RETRY_TIMEOUT,
+                              CONNECT_RETRY_TIMEOUT,
                               &try_connect_using_address,
                               cstate);
 }
diff --git a/src/util/disk.c b/src/util/disk.c
index d3d5d87dc..d536ec897 100644
--- a/src/util/disk.c
+++ b/src/util/disk.c
@@ -329,8 +329,10 @@ GNUNET_DISK_file_get_identifiers (const char *filename, uint64_t * dev,
     BY_HANDLE_FILE_INFORMATION info;
     int succ;
 
-    fh = GNUNET_DISK_file_open (filename, GNUNET_DISK_OPEN_READ, 0);
-    if (fh == NULL)
+    fh = GNUNET_DISK_file_open (filename,
+                                GNUNET_DISK_OPEN_READ,
+                                GNUNET_DISK_PERM_NONE);
+    if (NULL == fh)
       return GNUNET_SYSERR;
     succ = GetFileInformationByHandle (fh->h, &info);
     GNUNET_DISK_file_close (fh);
@@ -1191,7 +1193,7 @@ GNUNET_DISK_fn_write (const char *fn,
   fh = GNUNET_DISK_file_open (fn,
                               GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_TRUNCATE
                               | GNUNET_DISK_OPEN_CREATE, mode);
-  if (!fh)
+  if (! fh)
     return GNUNET_SYSERR;
   ret = GNUNET_DISK_file_write (fh, buffer, n);
   GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_close (fh));
@@ -1756,9 +1758,10 @@ GNUNET_DISK_file_open (const char *fn,
 
 
 /**
- * Close an open file
+ * Close an open file.
+ *
  * @param h file handle
- * @return GNUNET_OK on success, GNUNET_SYSERR otherwise
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR otherwise
  */
 int
 GNUNET_DISK_file_close (struct GNUNET_DISK_FileHandle *h)
@@ -1773,7 +1776,7 @@ GNUNET_DISK_file_close (struct GNUNET_DISK_FileHandle *h)
   ret = GNUNET_OK;
 
 #if MINGW
-  if (!CloseHandle (h->h))
+  if (! CloseHandle (h->h))
   {
     SetErrnoFromWinError (GetLastError ());
     LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "close");
@@ -1781,7 +1784,7 @@ GNUNET_DISK_file_close (struct GNUNET_DISK_FileHandle *h)
   }
   if (h->oOverlapRead)
   {
-    if (!CloseHandle (h->oOverlapRead->hEvent))
+    if (! CloseHandle (h->oOverlapRead->hEvent))
     {
       SetErrnoFromWinError (GetLastError ());
       LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "close");
@@ -1822,7 +1825,6 @@ struct GNUNET_DISK_FileHandle *
 GNUNET_DISK_get_handle_from_w32_handle (HANDLE osfh)
 {
   struct GNUNET_DISK_FileHandle *fh;
-
   DWORD dwret;
   enum GNUNET_FILE_Type ftype;
 
@@ -1836,7 +1838,8 @@ GNUNET_DISK_get_handle_from_w32_handle (HANDLE osfh)
     ftype = GNUNET_DISK_HANLDE_TYPE_PIPE;
     break;
   case FILE_TYPE_UNKNOWN:
-    if (GetLastError () == NO_ERROR || GetLastError () == ERROR_INVALID_HANDLE)
+    if ( (GetLastError () == NO_ERROR) ||
+         (GetLastError () == ERROR_INVALID_HANDLE) )
     {
       if (0 != ResetEvent (osfh))
         ftype = GNUNET_DISK_HANLDE_TYPE_EVENT;
diff --git a/src/util/getopt.c b/src/util/getopt.c
index ff62dba9b..036e0f4be 100644
--- a/src/util/getopt.c
+++ b/src/util/getopt.c
@@ -26,7 +26,7 @@ USA.
 
 
 This code was heavily modified for GNUnet.
-Copyright Copyright (C) 2006 Christian Grothoff
+Copyright Copyright (C) 2006, 2017 Christian Grothoff
 */
 
 /**
@@ -845,9 +845,13 @@ GN_getopt_internal (int argc, char *const *argv, const char *optstring,
   }
 }
 
+
 static int
-GNgetopt_long (int argc, char *const *argv, const char *options,
-               const struct GNoption *long_options, int *opt_index)
+GNgetopt_long (int argc,
+               char *const *argv,
+               const char *options,
+               const struct GNoption *long_options,
+               int *opt_index)
 {
   return GN_getopt_internal (argc, argv, options, long_options, opt_index, 0);
 }
@@ -867,16 +871,17 @@ GNgetopt_long (int argc, char *const *argv, const char *options,
 int
 GNUNET_GETOPT_run (const char *binaryOptions,
                    const struct GNUNET_GETOPT_CommandLineOption *allOptions,
-                   unsigned int argc, char *const *argv)
+                   unsigned int argc,
+                   char *const *argv)
 {
   struct GNoption *long_options;
   struct GNUNET_GETOPT_CommandLineProcessorContext clpc;
   int count;
-  int i;
   char *shorts;
   int spos;
   int cont;
   int c;
+  uint8_t *seen;
 
   GNUNET_assert (argc > 0);
   GNoptind = 0;
@@ -885,13 +890,15 @@ GNUNET_GETOPT_run (const char *binaryOptions,
   clpc.allOptions = allOptions;
   clpc.argv = argv;
   clpc.argc = argc;
-  count = 0;
-  while (allOptions[count].name != NULL)
-    count++;
-  long_options = GNUNET_malloc (sizeof (struct GNoption) * (count + 1));
+  for (count = 0; NULL != allOptions[count].name; count++) ;
+
+  long_options = GNUNET_new_array (count + 1,
+                                   struct GNoption);
+  seen = GNUNET_new_array (count,
+                           uint8_t);
   shorts = GNUNET_malloc (count * 2 + 1);
   spos = 0;
-  for (i = 0; i < count; i++)
+  for (unsigned i = 0; i < count; i++)
   {
     long_options[i].name = allOptions[i].name;
     long_options[i].has_arg = allOptions[i].require_argument;
@@ -907,13 +914,17 @@ GNUNET_GETOPT_run (const char *binaryOptions,
   long_options[count].val = '\0';
   shorts[spos] = '\0';
   cont = GNUNET_OK;
+
   /* main getopt loop */
-  while (cont == GNUNET_OK)
+  while (GNUNET_OK == cont)
   {
     int option_index = 0;
+    unsigned int i;
 
-    c = GNgetopt_long (argc, argv, shorts, long_options, &option_index);
-
+    c = GNgetopt_long (argc, argv,
+                       shorts,
+                       long_options,
+                       &option_index);
     if (c == GNUNET_SYSERR)
       break;                    /* No more flags to process */
 
@@ -922,25 +933,46 @@ GNUNET_GETOPT_run (const char *binaryOptions,
       clpc.currentArgument = GNoptind - 1;
       if ((char) c == allOptions[i].shortName)
       {
-        cont =
-            allOptions[i].processor (&clpc, allOptions[i].scls,
-                                     allOptions[i].name, GNoptarg);
+        cont = allOptions[i].processor (&clpc,
+                                        allOptions[i].scls,
+                                        allOptions[i].name,
+                                        GNoptarg);
+        seen[i] = 1;
         break;
       }
     }
     if (i == count)
     {
-      FPRINTF (stderr, _("Use %s to get a list of options.\n"), "--help");
+      FPRINTF (stderr,
+               _("Use %s to get a list of options.\n"),
+               "--help");
       cont = GNUNET_SYSERR;
     }
   }
-
   GNUNET_free (shorts);
   GNUNET_free (long_options);
-  if (cont != GNUNET_OK)
+
+  if (GNUNET_YES == cont)
   {
-    return cont;
+    for (count = 0; NULL != allOptions[count].name; count++)
+      if ( (0 == seen[count]) &&
+           (allOptions[count].option_mandatory) )
+      {
+        FPRINTF (stderr,
+                 _("Missing mandatory option `%s'.\n"),
+                 allOptions[count].name);
+        cont = GNUNET_SYSERR;
+      }
   }
+  GNUNET_free (seen);
+
+  /* call cleaners, if available */
+  for (count = 0; NULL != allOptions[count].name; count++)
+    if (NULL != allOptions[count].cleaner)
+      allOptions[count].cleaner (allOptions[count].scls);
+
+  if (GNUNET_OK != cont)
+    return cont;
   return GNoptind;
 }
 
diff --git a/src/util/getopt_helpers.c b/src/util/getopt_helpers.c
index 4d7104503..76342a6c9 100644
--- a/src/util/getopt_helpers.c
+++ b/src/util/getopt_helpers.c
@@ -38,11 +38,11 @@
  * @param value not used (NULL)
  * @return #GNUNET_NO (do not continue, not an error)
  */
-int
-GNUNET_GETOPT_print_version_ (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                              void *scls,
-                              const char *option,
-                              const char *value)
+static int
+print_version (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+               void *scls,
+               const char *option,
+               const char *value)
 {
   const char *version = scls;
 
@@ -54,6 +54,26 @@ GNUNET_GETOPT_print_version_ (struct GNUNET_GETOPT_CommandLineProcessorContext *
 
 
 /**
+ * Define the option to print the version of
+ * the application (-v option)
+ *
+ * @param version string with the version number
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_VERSION (const char *version)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  'v',
+    .name = "version",
+    .description = gettext_noop("print the version number"),
+    .processor = &print_version,
+    .scls = (void *) version
+  };
+  return clo;
+}
+
+
+/**
  * At what offset does the help text start?
  */
 #define BORDER 29
@@ -67,11 +87,11 @@ GNUNET_GETOPT_print_version_ (struct GNUNET_GETOPT_CommandLineProcessorContext *
  * @param value not used (NULL)
  * @return #GNUNET_NO (do not continue, not an error)
  */
-int
-GNUNET_GETOPT_format_help_ (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                            void *scls,
-                            const char *option,
-                            const char *value)
+static int
+format_help (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+             void *scls,
+             const char *option,
+             const char *value)
 {
   const char *about = scls;
   size_t slen;
@@ -165,6 +185,27 @@ OUTER:
 
 
 /**
+ * Defining the option to print the command line
+ * help text (-h option).
+ *
+ * @param about string with brief description of the application
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_HELP (const char *about)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = 'h',
+    .name = "help",
+    .description = gettext_noop("print this help"),
+    .processor = format_help,
+    .scls = (void *) about
+  };
+
+  return clo;
+}
+
+
+/**
  * Set an option of type 'unsigned int' from the command line. Each
  * time the option flag is given, the value is incremented by one.
  * A pointer to this function should be passed as part of the
@@ -173,17 +214,18 @@ OUTER:
  * type 'int'.
  *
  * @param ctx command line processing context
- * @param scls additional closure (will point to the 'int')
+ * @param scls additional closure (will point to the 'unsigned int')
  * @param option name of the option
  * @param value not used (NULL)
  * @return #GNUNET_OK
  */
-int
-GNUNET_GETOPT_increment_value (struct GNUNET_GETOPT_CommandLineProcessorContext
-                               *ctx, void *scls, const char *option,
-                               const char *value)
+static int
+increment_value (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+                 void *scls,
+                 const char *option,
+                 const char *value)
 {
-  int *val = scls;
+  unsigned int *val = scls;
 
   (*val)++;
   return GNUNET_OK;
@@ -191,6 +233,54 @@ GNUNET_GETOPT_increment_value (struct GNUNET_GETOPT_CommandLineProcessorContext
 
 
 /**
+ * Increment @a val each time the option flag is given by one.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val increment by 1 each time the option is present
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_INCREMENT_VALUE (char shortName,
+                                      const char *name,
+                                      const char *description,
+                                      unsigned int *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .description = description,
+    .processor = &increment_value,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
+ * Define the '-V' verbosity option.  Using the option more
+ * than once increments @a level each time.
+ *
+ * @param[out] level set to the verbosity level
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_VERBOSE (unsigned int *level)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = 'V',
+    .name = "verbose",
+    .description = gettext_noop("be verbose"),
+    .processor = &increment_value,
+    .scls = (void *) level
+  };
+
+  return clo;
+}
+
+
+/**
  * Set an option of type 'int' from the command line to 1 if the
  * given option is present.
  * A pointer to this function should be passed as part of the
@@ -204,9 +294,11 @@ GNUNET_GETOPT_increment_value (struct GNUNET_GETOPT_CommandLineProcessorContext
  * @param value not used (NULL)
  * @return #GNUNET_OK
  */
-int
-GNUNET_GETOPT_set_one (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                       void *scls, const char *option, const char *value)
+static int
+set_one (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+         void *scls,
+         const char *option,
+         const char *value)
 {
   int *val = scls;
 
@@ -216,6 +308,34 @@ GNUNET_GETOPT_set_one (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
 
 
 /**
+ * Allow user to specify a flag (which internally means setting
+ * an integer to 1/#GNUNET_YES/#GNUNET_OK.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to 1 if the option is present
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_ONE (char shortName,
+                              const char *name,
+                              const char *description,
+                              int *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .description = description,
+    .processor = &set_one,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
  * Set an option of type 'char *' from the command line.
  * A pointer to this function should be passed as part of the
  * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
@@ -229,31 +349,174 @@ GNUNET_GETOPT_set_one (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
  * @param value actual value of the option (a string)
  * @return #GNUNET_OK
  */
-int
-GNUNET_GETOPT_set_string (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                          void *scls, const char *option, const char *value)
+static int
+set_string (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+            void *scls,
+            const char *option,
+            const char *value)
 {
   char **val = scls;
 
-  GNUNET_assert (value != NULL);
+  GNUNET_assert (NULL != value);
   GNUNET_free_non_null (*val);
   *val = GNUNET_strdup (value);
   return GNUNET_OK;
 }
 
 
-int
-GNUNET_GETOPT_set_filename (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                            void *scls, const char *option, const char *value)
+/**
+ * Allow user to specify a string.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] str set to the string
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_STRING (char shortName,
+                             const char *name,
+                             const char *argumentHelp,
+                             const char *description,
+                             char **str)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_string,
+    .scls = (void *) str
+  };
+
+  return clo;
+}
+
+
+/**
+ * Define the '-L' log level option.  Note that we do not check
+ * that the log level is valid here.
+ *
+ * @param[out] level set to the log level
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_LOGLEVEL (char **level)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName = 'L',
+    .name = "log",
+    .argumentHelp = "LOGLEVEL",
+    .description = gettext_noop("configure logging to use LOGLEVEL"),
+    .require_argument = 1,
+    .processor = &set_string,
+    .scls = (void *) level
+  };
+
+  return clo;
+}
+
+
+/**
+ * Set an option of type 'char *' from the command line with
+ * filename expansion a la #GNUNET_STRINGS_filename_expand().
+ *
+ * @param ctx command line processing context
+ * @param scls additional closure (will point to the `char *`,
+ *             which will be allocated)
+ * @param option name of the option
+ * @param value actual value of the option (a string)
+ * @return #GNUNET_OK
+ */
+static int
+set_filename (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+              void *scls,
+              const char *option,
+              const char *value)
 {
   char **val = scls;
 
-  GNUNET_assert (value != NULL);
+  GNUNET_assert (NULL != value);
   GNUNET_free_non_null (*val);
   *val = GNUNET_STRINGS_filename_expand (value);
   return GNUNET_OK;
 }
 
+
+/**
+ * Allow user to specify a filename (automatically path expanded).
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] str set to the string
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_FILENAME (char shortName,
+                             const char *name,
+                             const char *argumentHelp,
+                             const char *description,
+                             char **str)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_filename,
+    .scls = (void *) str
+  };
+
+  return clo;
+}
+
+
+/**
+ * Allow user to specify log file name (-l option)
+ *
+ * @param[out] logfn set to the name of the logfile
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_LOGFILE (char **logfn)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  'l',
+    .name = "logfile",
+    .argumentHelp = "FILENAME",
+    .description = gettext_noop ("configure logging to write logs to FILENAME"),
+    .require_argument = 1,
+    .processor = &set_filename,
+    .scls = (void *) logfn
+  };
+
+  return clo;
+}
+
+
+/**
+ * Allow user to specify configuration file name (-c option)
+ *
+ * @param[out] fn set to the name of the configuration file
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_CFG_FILE (char **fn)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  'c',
+    .name = "config",
+    .argumentHelp = "FILENAME",
+    .description = gettext_noop("use configuration file FILENAME"),
+    .require_argument = 1,
+    .processor = &set_filename,
+    .scls = (void *) fn
+  };
+
+  return clo;
+}
+
+
 /**
  * Set an option of type 'unsigned long long' from the command line.
  * A pointer to this function should be passed as part of the
@@ -267,15 +530,21 @@ GNUNET_GETOPT_set_filename (struct GNUNET_GETOPT_CommandLineProcessorContext *ct
  * @param value actual value of the option as a string.
  * @return #GNUNET_OK if parsing the value worked
  */
-int
-GNUNET_GETOPT_set_ulong (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                         void *scls, const char *option, const char *value)
+static int
+set_ulong (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+           void *scls,
+           const char *option,
+           const char *value)
 {
   unsigned long long *val = scls;
 
-  if (1 != SSCANF (value, "%llu", val))
+  if (1 != SSCANF (value,
+                   "%llu",
+                   val))
   {
-    FPRINTF (stderr, _("You must pass a number to the `%s' option.\n"), option);
+    FPRINTF (stderr,
+             _("You must pass a number to the `%s' option.\n"),
+             option);
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -283,6 +552,36 @@ GNUNET_GETOPT_set_ulong (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
 
 
 /**
+ * Allow user to specify an `unsigned long long`
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the value specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_ULONG (char shortName,
+                                const char *name,
+                                const char *argumentHelp,
+                                const char *description,
+                                unsigned long long *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_ulong,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
  * Set an option of type 'struct GNUNET_TIME_Relative' from the command line.
  * A pointer to this function should be passed as part of the
  * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
@@ -295,9 +594,11 @@ GNUNET_GETOPT_set_ulong (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
  * @param value actual value of the option as a string.
  * @return #GNUNET_OK if parsing the value worked
  */
-int
-GNUNET_GETOPT_set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                                 void *scls, const char *option, const char *value)
+static int
+set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+                   void *scls,
+                   const char *option,
+                   const char *value)
 {
   struct GNUNET_TIME_Relative *val = scls;
 
@@ -305,7 +606,74 @@ GNUNET_GETOPT_set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContex
       GNUNET_STRINGS_fancy_time_to_relative (value,
                                              val))
   {
-    FPRINTF (stderr, _("You must pass relative time to the `%s' option.\n"), option);
+    FPRINTF (stderr,
+             _("You must pass relative time to the `%s' option.\n"),
+             option);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Allow user to specify a `struct GNUNET_TIME_Relative`
+ * (using human-readable "fancy" time).
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the time specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME (char shortName,
+                                        const char *name,
+                                        const char *argumentHelp,
+                                        const char *description,
+                                        struct GNUNET_TIME_Relative *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_relative_time,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
+ * Set an option of type 'struct GNUNET_TIME_Absolute' from the command line.
+ * A pointer to this function should be passed as part of the
+ * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
+ * of this type.  It should be followed by a pointer to a value of
+ * type 'struct GNUNET_TIME_Absolute'.
+ *
+ * @param ctx command line processing context
+ * @param scls additional closure (will point to the `struct GNUNET_TIME_Absolute`)
+ * @param option name of the option
+ * @param value actual value of the option as a string.
+ * @return #GNUNET_OK if parsing the value worked
+ */
+static int
+set_absolute_time (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+                   void *scls,
+                   const char *option,
+                   const char *value)
+{
+  struct GNUNET_TIME_Absolute *val = scls;
+
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_fancy_time_to_absolute (value,
+                                             val))
+  {
+    FPRINTF (stderr,
+             _("You must pass absolute time to the `%s' option.\n"),
+             option);
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
@@ -313,6 +681,37 @@ GNUNET_GETOPT_set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContex
 
 
 /**
+ * Allow user to specify a `struct GNUNET_TIME_Absolute`
+ * (using human-readable "fancy" time).
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the time specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_ABSOLUTE_TIME (char shortName,
+                                        const char *name,
+                                        const char *argumentHelp,
+                                        const char *description,
+                                        struct GNUNET_TIME_Absolute *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_absolute_time,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
  * Set an option of type 'unsigned int' from the command line.
  * A pointer to this function should be passed as part of the
  * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
@@ -325,19 +724,172 @@ GNUNET_GETOPT_set_relative_time (struct GNUNET_GETOPT_CommandLineProcessorContex
  * @param value actual value of the option as a string.
  * @return #GNUNET_OK if parsing the value worked
  */
-int
-GNUNET_GETOPT_set_uint (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
-                        void *scls, const char *option, const char *value)
+static int
+set_uint (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+          void *scls,
+          const char *option,
+          const char *value)
 {
   unsigned int *val = scls;
 
-  if (1 != SSCANF (value, "%u", val))
+  if (1 != SSCANF (value,
+                   "%u",
+                   val))
   {
-    FPRINTF (stderr, _("You must pass a number to the `%s' option.\n"), option);
+    FPRINTF (stderr,
+             _("You must pass a number to the `%s' option.\n"),
+             option);
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
 }
 
 
+/**
+ * Allow user to specify an unsigned integer.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val set to the value specified at the command line
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_UINT (char shortName,
+                               const char *name,
+                               const char *argumentHelp,
+                               const char *description,
+                               unsigned int *val)
+{
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_uint,
+    .scls = (void *) val
+  };
+
+  return clo;
+}
+
+
+/**
+ * Closure for #set_base32().
+ */
+struct Base32Context
+{
+  /**
+   * Value to initialize (already allocated)
+   */
+  void *val;
+
+  /**
+   * Number of bytes expected for @e val.
+   */
+  size_t val_size;
+};
+
+
+/**
+ * Set an option of type 'unsigned int' from the command line.
+ * A pointer to this function should be passed as part of the
+ * 'struct GNUNET_GETOPT_CommandLineOption' array to initialize options
+ * of this type.  It should be followed by a pointer to a value of
+ * type 'unsigned int'.
+ *
+ * @param ctx command line processing context
+ * @param scls additional closure (will point to the 'unsigned int')
+ * @param option name of the option
+ * @param value actual value of the option as a string.
+ * @return #GNUNET_OK if parsing the value worked
+ */
+static int
+set_base32 (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx,
+            void *scls,
+            const char *option,
+            const char *value)
+{
+  struct Base32Context *bc = scls;
+
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_string_to_data (value,
+                                     strlen (value),
+                                     bc->val,
+                                     bc->val_size))
+  {
+    fprintf (stderr,
+             _("Argument `%s' malformed. Expected base32 (Crockford) encoded value.\n"),
+             option);
+    return GNUNET_SYSERR;
+  }
+  return GNUNET_OK;
+}
+
+
+/**
+ * Helper function to clean up after
+ * #GNUNET_GETOPT_OPTION_SET_BASE32_FIXED_SIZE.
+ *
+ * @param cls value to GNUNET_free()
+ */
+static void
+free_bc (void *cls)
+{
+  GNUNET_free (cls);
+}
+
+
+/**
+ * Allow user to specify a binary value using Crockford
+ * Base32 encoding.
+ *
+ * @param shortName short name of the option
+ * @param name long name of the option
+ * @param argumentHelp help text for the option argument
+ * @param description long help text for the option
+ * @param[out] val binary value decoded from Crockford Base32-encoded argument
+ * @param val_size size of @a val in bytes
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_SET_BASE32_FIXED_SIZE (char shortName,
+                                            const char *name,
+                                            const char *argumentHelp,
+                                            const char *description,
+                                            void *val,
+                                            size_t val_size)
+{
+  struct Base32Context *bc = GNUNET_new (struct Base32Context);
+  struct GNUNET_GETOPT_CommandLineOption clo = {
+    .shortName =  shortName,
+    .name = name,
+    .argumentHelp = argumentHelp,
+    .description = description,
+    .require_argument = 1,
+    .processor = &set_base32,
+    .cleaner = &free_bc,
+    .scls = (void *) bc
+  };
+
+  bc->val = val;
+  bc->val_size = val_size;
+  return clo;
+}
+
+
+/**
+ * Make the given option mandatory.
+ *
+ * @param opt option to modify
+ * @return @a opt with the mandatory flag set.
+ */
+struct GNUNET_GETOPT_CommandLineOption
+GNUNET_GETOPT_OPTION_MANDATORY (struct GNUNET_GETOPT_CommandLineOption opt)
+{
+  opt.option_mandatory = 1;
+  return opt;
+}
+
+
 /* end of getopt_helpers.c */
diff --git a/src/util/gnunet-config.c b/src/util/gnunet-config.c
index 7ec7162f1..2beb772a9 100644
--- a/src/util/gnunet-config.c
+++ b/src/util/gnunet-config.c
@@ -223,34 +223,48 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'f', "filename", NULL,
-      gettext_noop ("obtain option of value as a filename (with $-expansion)"),
-      0, &GNUNET_GETOPT_set_one, &is_filename },
-    { 's', "section", "SECTION",
-      gettext_noop ("name of the section to access"),
-      1, &GNUNET_GETOPT_set_string, &section },
-    { 'o', "option", "OPTION",
-      gettext_noop ("name of the option to access"),
-      1, &GNUNET_GETOPT_set_string, &option },
-    { 'V', "value", "VALUE",
-      gettext_noop ("value to set"),
-      1, &GNUNET_GETOPT_set_string, &value },
-    { 'S', "list-sections", NULL,
-      gettext_noop ("print available configuration sections"),
-      0, &GNUNET_GETOPT_set_one, &list_sections },
-    { 'w', "rewrite", NULL,
-      gettext_noop ("write configuration file that only contains delta to defaults"),
-      0, &GNUNET_GETOPT_set_one, &rewrite },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('f',
+                                  "filename",
+                                  gettext_noop ("obtain option of value as a filename (with $-expansion)"),
+                                  &is_filename),
+    GNUNET_GETOPT_OPTION_STRING ('s',
+                                     "section",
+                                     "SECTION",
+                                     gettext_noop ("name of the section to access"),
+                                     &section),
+    GNUNET_GETOPT_OPTION_STRING ('o',
+                                     "option",
+                                     "OPTION",
+                                     gettext_noop ("name of the option to access"),
+                                     &option),
+    GNUNET_GETOPT_OPTION_STRING ('V',
+                                     "value",
+                                     "VALUE",
+                                     gettext_noop ("value to set"),
+                                     &value),
+    GNUNET_GETOPT_OPTION_SET_ONE ('S',
+                                  "list-sections",
+                                  gettext_noop ("print available configuration sections"),
+                                  &list_sections),
+    GNUNET_GETOPT_OPTION_SET_ONE ('w',
+                                  "rewrite",
+                                  gettext_noop ("write configuration file that only contains delta to defaults"),
+                                  &rewrite),
     GNUNET_GETOPT_OPTION_END
   };
-  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_get_utf8_args (argc, argv,
+                                    &argc, &argv))
     return 2;
 
   ret = (GNUNET_OK ==
-         GNUNET_PROGRAM_run (argc, argv, "gnunet-config [OPTIONS]",
+         GNUNET_PROGRAM_run (argc,
+                             argv,
+                             "gnunet-config [OPTIONS]",
                              gettext_noop ("Manipulate GNUnet configuration files"),
-                             options, &run, NULL)) ? 0 : ret;
+                             options,
+                             &run, NULL)) ? 0 : ret;
   GNUNET_free ((void*) argv);
   return ret;
 }
diff --git a/src/util/gnunet-ecc.c b/src/util/gnunet-ecc.c
index ddfd9b1c3..732228b52 100644
--- a/src/util/gnunet-ecc.c
+++ b/src/util/gnunet-ecc.c
@@ -41,7 +41,7 @@ static int list_keys;
 /**
  * Flag for listing public key.
  */
-static int list_keys_count;
+static unsigned int list_keys_count;
 
 /**
  * Flag for printing public key.
@@ -406,36 +406,50 @@ run (void *cls, char *const *args, const char *cfgfile,
  * @return 0 ok, 1 on error
  */
 int
-main (int argc, char *const *argv)
+main (int argc,
+      char *const *argv)
 {
   list_keys_count = UINT32_MAX;
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'i', "iterate", "FILE",
-      gettext_noop ("list keys included in a file (for testing)"),
-      0, &GNUNET_GETOPT_set_one, &list_keys },
-    { 'e', "end=", "COUNT",
-      gettext_noop ("number of keys to list included in a file (for testing)"),
-      1, &GNUNET_GETOPT_set_uint, &list_keys_count },
-    { 'g', "generate-keys", "COUNT",
-      gettext_noop ("create COUNT public-private key pairs (for testing)"),
-      1, &GNUNET_GETOPT_set_uint, &make_keys },
-    { 'p', "print-public-key", NULL,
-      gettext_noop ("print the public key in ASCII format"),
-      0, &GNUNET_GETOPT_set_one, &print_public_key },
-    { 'E', "examples", NULL,
-      gettext_noop ("print examples of ECC operations (used for compatibility testing)"),
-      0, &GNUNET_GETOPT_set_one, &print_examples_flag },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('i',
+                                  "iterate",
+                                  gettext_noop ("list keys included in a file (for testing)"),
+                                  &list_keys),
+    GNUNET_GETOPT_OPTION_SET_UINT ('e',
+                                   "end=",
+                                   "COUNT",
+                                   gettext_noop ("number of keys to list included in a file (for testing)"),
+                                   &list_keys_count),
+    GNUNET_GETOPT_OPTION_SET_UINT ('g',
+                                   "generate-keys",
+                                   "COUNT",
+                                   gettext_noop ("create COUNT public-private key pairs (for testing)"),
+                                   &make_keys),
+    GNUNET_GETOPT_OPTION_SET_ONE ('p',
+                                  "print-public-key",
+                                  gettext_noop ("print the public key in ASCII format"),
+                                  &print_public_key),
+    GNUNET_GETOPT_OPTION_SET_ONE ('E',
+                                  "examples",
+                                  gettext_noop ("print examples of ECC operations (used for compatibility testing)"),
+                                  &print_examples_flag),
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
 
-  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
+  if (GNUNET_OK !=
+      GNUNET_STRINGS_get_utf8_args (argc, argv,
+                                    &argc, &argv))
     return 2;
 
   ret = (GNUNET_OK ==
-         GNUNET_PROGRAM_run (argc, argv, "gnunet-ecc [OPTIONS] keyfile [VANITY_PREFIX]",
+         GNUNET_PROGRAM_run (argc,
+                             argv,
+                             "gnunet-ecc [OPTIONS] keyfile [VANITY_PREFIX]",
                              gettext_noop ("Manipulate GNUnet private ECC key files"),
-                             options, &run, NULL)) ? 0 : 1;
+                             options,
+                             &run,
+                             NULL)) ? 0 : 1;
   GNUNET_free ((void*) argv);
   return ret;
 }
diff --git a/src/util/gnunet-resolver.c b/src/util/gnunet-resolver.c
index e84a2332f..563cf9fce 100644
--- a/src/util/gnunet-resolver.c
+++ b/src/util/gnunet-resolver.c
@@ -144,10 +144,11 @@ run (void *cls, char *const *args, const char *cfgfile,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'r', "reverse", NULL,
-      gettext_noop ("perform a reverse lookup"),
-      0, &GNUNET_GETOPT_set_one, &reverse },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('r',
+                                  "reverse",
+                                  gettext_noop ("perform a reverse lookup"),
+                                  &reverse),
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
diff --git a/src/util/gnunet-scrypt.c b/src/util/gnunet-scrypt.c
index ab0cf92e0..7c73cfe1e 100644
--- a/src/util/gnunet-scrypt.c
+++ b/src/util/gnunet-scrypt.c
@@ -307,21 +307,30 @@ run (void *cls,
  * @return 0 ok, 1 on error
  */
 int
-main (int argc, char *const *argv)
+main (int argc,
+      char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    { 'b', "bits", "BITS",
-      gettext_noop ("number of bits to require for the proof of work"),
-      1, &GNUNET_GETOPT_set_ulong, &nse_work_required },
-    { 'k', "keyfile", "FILE",
-      gettext_noop ("file with private key, otherwise default is used"),
-      1, &GNUNET_GETOPT_set_filename, &pkfn },
-    { 'o', "outfile", "FILE",
-      gettext_noop ("file with proof of work, otherwise default is used"),
-      1, &GNUNET_GETOPT_set_filename, &pwfn },
-    { 't', "timeout", "TIME",
-      gettext_noop ("time to wait between calculations"),
-      1, &GNUNET_GETOPT_set_relative_time, &proof_find_delay },
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ULONG ('b',
+                                    "bits",
+                                    "BITS",
+                                    gettext_noop ("number of bits to require for the proof of work"),
+                                    &nse_work_required),
+    GNUNET_GETOPT_OPTION_FILENAME ('k',
+                                   "keyfile",
+                                   "FILE",
+                                   gettext_noop ("file with private key, otherwise default is used"),
+                                   &pkfn),
+    GNUNET_GETOPT_OPTION_FILENAME ('o',
+                                   "outfile",
+                                   "FILE",
+                                   gettext_noop ("file with proof of work, otherwise default is used"),
+                                   &pwfn),
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('t',
+                                            "timeout",
+                                            "TIME",
+                                            gettext_noop ("time to wait between calculations"),
+                                            &proof_find_delay),
     GNUNET_GETOPT_OPTION_END
   };
   int ret;
@@ -334,7 +343,9 @@ main (int argc, char *const *argv)
          GNUNET_PROGRAM_run (argc, argv,
                              "gnunet-scrypt [OPTIONS] prooffile",
                              gettext_noop ("Manipulate GNUnet proof of work files"),
-                             options, &run, NULL)) ? 0 : 1;
+                             options,
+                             &run,
+                             NULL)) ? 0 : 1;
   GNUNET_free ((void*) argv);
   GNUNET_free_non_null (pwfn);
   return ret;
diff --git a/src/util/helper.c b/src/util/helper.c
index cdb1b01d4..a84b06e66 100644
--- a/src/util/helper.c
+++ b/src/util/helper.c
@@ -27,6 +27,7 @@
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
+#include "gnunet_mst_lib.h"
 
 
 /**
@@ -107,7 +108,7 @@ struct GNUNET_HELPER_Handle
   /**
    * The Message-Tokenizer that tokenizes the messages comming from the helper
    */
-  struct GNUNET_SERVER_MessageStreamTokenizer *mst;
+  struct GNUNET_MessageStreamTokenizer *mst;
 
   /**
    * The exception callback
@@ -272,7 +273,10 @@ GNUNET_HELPER_wait (struct GNUNET_HELPER_Handle *h)
   }
   /* purge MST buffer */
   if (NULL != h->mst)
-    (void) GNUNET_SERVER_mst_receive (h->mst, NULL, NULL, 0, GNUNET_YES, GNUNET_NO);
+    (void) GNUNET_MST_from_buffer (h->mst,
+                                   NULL, 0,
+                                   GNUNET_YES,
+                                   GNUNET_NO);
   return ret;
 }
 
@@ -319,7 +323,7 @@ static void
 helper_read (void *cls)
 {
   struct GNUNET_HELPER_Handle *h = cls;
-  char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE] GNUNET_ALIGN;
+  char buf[GNUNET_MAX_MESSAGE_SIZE] GNUNET_ALIGN;
   ssize_t t;
 
   h->read_task = NULL;
@@ -373,10 +377,10 @@ helper_read (void *cls)
                                                  h->fh_from_helper,
                                                  &helper_read, h);
   if (GNUNET_SYSERR ==
-      GNUNET_SERVER_mst_receive (h->mst,
-                                 NULL,
-                                 buf, t,
-                                 GNUNET_NO, GNUNET_NO))
+      GNUNET_MST_from_buffer (h->mst,
+                              buf, t,
+                              GNUNET_NO,
+                              GNUNET_NO))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 _("Failed to parse inbound message from helper `%s'\n"),
@@ -487,7 +491,7 @@ struct GNUNET_HELPER_Handle *
 GNUNET_HELPER_start (int with_control_pipe,
                      const char *binary_name,
                      char *const binary_argv[],
-                     GNUNET_SERVER_MessageTokenizerCallback cb,
+                     GNUNET_MessageTokenizerCallback cb,
                      GNUNET_HELPER_ExceptionCallback exp_cb,
                      void *cb_cls)
 {
@@ -508,7 +512,8 @@ GNUNET_HELPER_start (int with_control_pipe,
   h->binary_argv[c] = NULL;
   h->cb_cls = cb_cls;
   if (NULL != cb)
-    h->mst = GNUNET_SERVER_mst_create (cb, h->cb_cls);
+    h->mst = GNUNET_MST_create (cb,
+                                h->cb_cls);
   h->exp_cb = exp_cb;
   h->retry_back_off = 0;
   start_helper (h);
@@ -544,7 +549,7 @@ GNUNET_HELPER_destroy (struct GNUNET_HELPER_Handle *h)
     GNUNET_free (sh);
   }
   if (NULL != h->mst)
-    GNUNET_SERVER_mst_destroy (h->mst);
+    GNUNET_MST_destroy (h->mst);
   GNUNET_free (h->binary_name);
   for (c = 0; h->binary_argv[c] != NULL; c++)
     GNUNET_free (h->binary_argv[c]);
diff --git a/src/util/mq.c b/src/util/mq.c
index 79e2d0455..90b2aa968 100644
--- a/src/util/mq.c
+++ b/src/util/mq.c
@@ -202,24 +202,6 @@ struct GNUNET_MQ_Handle
 
 
 /**
- * Implementation-specific state for connection to
- * client (MQ for server).
- */
-struct ServerClientSocketState
-{
-  /**
-   * Handle of the client that connected to the server.
-   */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * Active transmission request to the client.
-   */
-  struct GNUNET_SERVER_TransmitHandle *th;
-};
-
-
-/**
  * Call the message message handler that was registered
  * for the type of the given message in the given message queue.
  *
@@ -708,92 +690,6 @@ GNUNET_MQ_msg_nested_mh_ (struct GNUNET_MessageHeader **mhp,
 
 
 /**
- * Transmit a queued message to the session's client.
- *
- * @param cls consensus session
- * @param size number of bytes available in @a buf
- * @param buf where the callee should write the message
- * @return number of bytes written to @a buf
- */
-static size_t
-transmit_queued (void *cls,
-                 size_t size,
-                 void *buf)
-{
-  struct GNUNET_MQ_Handle *mq = cls;
-  struct ServerClientSocketState *state = GNUNET_MQ_impl_state (mq);
-  const struct GNUNET_MessageHeader *msg = GNUNET_MQ_impl_current (mq);
-  size_t msg_size;
-
-  GNUNET_assert (NULL != buf);
-  msg_size = ntohs (msg->size);
-  GNUNET_assert (size >= msg_size);
-  GNUNET_memcpy (buf, msg, msg_size);
-  state->th = NULL;
-
-  GNUNET_MQ_impl_send_continue (mq);
-
-  return msg_size;
-}
-
-
-static void
-server_client_destroy_impl (struct GNUNET_MQ_Handle *mq,
-                            void *impl_state)
-{
-  struct ServerClientSocketState *state = impl_state;
-
-  if (NULL != state->th)
-  {
-    GNUNET_SERVER_notify_transmit_ready_cancel (state->th);
-    state->th = NULL;
-  }
-
-  GNUNET_assert (NULL != mq);
-  GNUNET_assert (NULL != state);
-  GNUNET_SERVER_client_drop (state->client);
-  GNUNET_free (state);
-}
-
-
-static void
-server_client_send_impl (struct GNUNET_MQ_Handle *mq,
-                         const struct GNUNET_MessageHeader *msg,
-                         void *impl_state)
-{
-  GNUNET_assert (NULL != mq);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending message of type %u and size %u\n",
-       ntohs (msg->type), ntohs (msg->size));
-
-  struct ServerClientSocketState *state = impl_state;
-  state->th = GNUNET_SERVER_notify_transmit_ready (state->client,
-                                                   ntohs (msg->size),
-                                                   GNUNET_TIME_UNIT_FOREVER_REL,
-                                                   &transmit_queued,
-                                                   mq);
-}
-
-
-struct GNUNET_MQ_Handle *
-GNUNET_MQ_queue_for_server_client (struct GNUNET_SERVER_Client *client)
-{
-  struct GNUNET_MQ_Handle *mq;
-  struct ServerClientSocketState *scss;
-
-  mq = GNUNET_new (struct GNUNET_MQ_Handle);
-  scss = GNUNET_new (struct ServerClientSocketState);
-  mq->impl_state = scss;
-  scss->client = client;
-  GNUNET_SERVER_client_keep (client);
-  mq->send_impl = &server_client_send_impl;
-  mq->destroy_impl = &server_client_destroy_impl;
-  return mq;
-}
-
-
-/**
  * Associate the assoc_data in mq with a unique request id.
  *
  * @param mq message queue, id will be unique for the queue
@@ -811,22 +707,40 @@ GNUNET_MQ_assoc_add (struct GNUNET_MQ_Handle *mq,
     mq->assoc_id = 1;
   }
   id = mq->assoc_id++;
-  GNUNET_CONTAINER_multihashmap32_put (mq->assoc_map, id, assoc_data,
-                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap32_put (mq->assoc_map,
+                                                      id,
+                                                      assoc_data,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   return id;
 }
 
 
+/**
+ * Get the data associated with a @a request_id in a queue
+ *
+ * @param mq the message queue with the association
+ * @param request_id the request id we are interested in
+ * @return the associated data
+ */
 void *
 GNUNET_MQ_assoc_get (struct GNUNET_MQ_Handle *mq,
                      uint32_t request_id)
 {
   if (NULL == mq->assoc_map)
     return NULL;
-  return GNUNET_CONTAINER_multihashmap32_get (mq->assoc_map, request_id);
+  return GNUNET_CONTAINER_multihashmap32_get (mq->assoc_map,
+                                              request_id);
 }
 
 
+/**
+ * Remove the association for a @a request_id
+ *
+ * @param mq the message queue with the association
+ * @param request_id the request id we want to remove
+ * @return the associated data
+ */
 void *
 GNUNET_MQ_assoc_remove (struct GNUNET_MQ_Handle *mq,
                         uint32_t request_id)
diff --git a/src/util/mst.c b/src/util/mst.c
index 9f1d30d7a..0d90c5d10 100644
--- a/src/util/mst.c
+++ b/src/util/mst.c
@@ -90,8 +90,8 @@ GNUNET_MST_create (GNUNET_MessageTokenizerCallback cb,
   struct GNUNET_MessageStreamTokenizer *ret;
 
   ret = GNUNET_new (struct GNUNET_MessageStreamTokenizer);
-  ret->hdr = GNUNET_malloc (GNUNET_SERVER_MIN_BUFFER_SIZE);
-  ret->curr_buf = GNUNET_SERVER_MIN_BUFFER_SIZE;
+  ret->hdr = GNUNET_malloc (GNUNET_MIN_MESSAGE_SIZE);
+  ret->curr_buf = GNUNET_MIN_MESSAGE_SIZE;
   ret->cb = cb;
   ret->cb_cls = cb_cls;
   return ret;
diff --git a/src/util/resolver_api.c b/src/util/resolver_api.c
index f33c31f1c..0c915932c 100644
--- a/src/util/resolver_api.c
+++ b/src/util/resolver_api.c
@@ -876,7 +876,7 @@ GNUNET_RESOLVER_ip_get (const char *hostname,
 
   slen = strlen (hostname) + 1;
   if (slen + sizeof (struct GNUNET_RESOLVER_GetMessage) >=
-      GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      GNUNET_MAX_MESSAGE_SIZE)
   {
     GNUNET_break (0);
     return NULL;
diff --git a/src/util/server_nc.c b/src/util/server_nc.c
deleted file mode 100644
index a95cd7f6d..000000000
--- a/src/util/server_nc.c
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2010 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file util/server_nc.c
- * @brief convenience functions for transmission of
- *        a notification stream
- * @author Christian Grothoff
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define LOG(kind,...) GNUNET_log_from (kind, "util-server-nc", __VA_ARGS__)
-
-
-/**
- * Entry in list of messages pending to be transmitted.
- */
-struct PendingMessageList
-{
-
-  /**
-   * This is a doubly-linked list.
-   */
-  struct PendingMessageList *next;
-
-  /**
-   * This is a doubly-linked list.
-   */
-  struct PendingMessageList *prev;
-
-  /**
-   * Message to transmit (allocated at the end of this
-   * struct, do not free)
-   */
-  const struct GNUNET_MessageHeader *msg;
-
-  /**
-   * Can this message be dropped?
-   */
-  int can_drop;
-
-};
-
-
-/**
- * Lists of clients we manage for notifications.
- */
-struct ClientList
-{
-
-  /**
-   * This is a doubly linked list.
-   */
-  struct ClientList *next;
-
-  /**
-   * This is a doubly linked list.
-   */
-  struct ClientList *prev;
-
-  /**
-   * Overall context this client belongs to.
-   */
-  struct GNUNET_SERVER_NotificationContext *nc;
-
-  /**
-   * Handle to the client.
-   */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * Handle for pending transmission request to the client (or NULL).
-   */
-  struct GNUNET_SERVER_TransmitHandle *th;
-
-  /**
-   * Head of linked list of requests queued for transmission.
-   */
-  struct PendingMessageList *pending_head;
-
-  /**
-   * Tail of linked list of requests queued for transmission.
-   */
-  struct PendingMessageList *pending_tail;
-
-  /**
-   * Number of messages currently in the list.
-   */
-  unsigned int num_pending;
-
-};
-
-
-/**
- * The notification context is the key datastructure for a convenience
- * API used for transmission of notifications to the client until the
- * client disconnects (or the notification context is destroyed, in
- * which case we disconnect these clients).  Essentially, all
- * (notification) messages are queued up until the client is able to
- * read them.
- */
-struct GNUNET_SERVER_NotificationContext
-{
-
-  /**
-   * Server we do notifications for.
-   */
-  struct GNUNET_SERVER_Handle *server;
-
-  /**
-   * Head of list of clients receiving notifications.
-   */
-  struct ClientList *clients_head;
-
-  /**
-   * Tail of list of clients receiving notifications.
-   */
-  struct ClientList *clients_tail;
-
-  /**
-   * Maximum number of optional messages to queue per client.
-   */
-  unsigned int queue_length;
-
-};
-
-
-/**
- * Client has disconnected, clean up.
- *
- * @param cls our `struct GNUNET_SERVER_NotificationContext *`
- * @param client handle of client that disconnected
- */
-static void
-handle_client_disconnect (void *cls,
-                          struct GNUNET_SERVER_Client *client)
-{
-  struct GNUNET_SERVER_NotificationContext *nc = cls;
-  struct ClientList *pos;
-  struct PendingMessageList *pml;
-
-  if (NULL == client)
-  {
-    nc->server = NULL;
-    return;
-  }
-  for (pos = nc->clients_head; NULL != pos; pos = pos->next)
-    if (pos->client == client)
-      break;
-  if (NULL == pos)
-    return;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Client disconnected, cleaning up %u messages in NC queue\n",
-       pos->num_pending);
-  GNUNET_CONTAINER_DLL_remove (nc->clients_head,
-                               nc->clients_tail,
-                               pos);
-  while (NULL != (pml = pos->pending_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (pos->pending_head,
-                                 pos->pending_tail,
-                                 pml);
-    GNUNET_free (pml);
-    pos->num_pending--;
-  }
-  if (NULL != pos->th)
-  {
-    GNUNET_SERVER_notify_transmit_ready_cancel (pos->th);
-    pos->th = NULL;
-  }
-  GNUNET_SERVER_client_drop (client);
-  GNUNET_assert (0 == pos->num_pending);
-  GNUNET_free (pos);
-}
-
-
-/**
- * Create a new notification context.
- *
- * @param server server for which this function creates the context
- * @param queue_length maximum number of messages to keep in
- *        the notification queue; optional messages are dropped
- *        if the queue gets longer than this number of messages
- * @return handle to the notification context
- */
-struct GNUNET_SERVER_NotificationContext *
-GNUNET_SERVER_notification_context_create (struct GNUNET_SERVER_Handle *server,
-                                           unsigned int queue_length)
-{
-  struct GNUNET_SERVER_NotificationContext *ret;
-
-  ret = GNUNET_new (struct GNUNET_SERVER_NotificationContext);
-  ret->server = server;
-  ret->queue_length = queue_length;
-  GNUNET_SERVER_disconnect_notify (server,
-                                   &handle_client_disconnect,
-                                   ret);
-  return ret;
-}
-
-
-/**
- * Destroy the context, force disconnect for all clients.
- *
- * @param nc context to destroy.
- */
-void
-GNUNET_SERVER_notification_context_destroy (struct GNUNET_SERVER_NotificationContext *nc)
-{
-  struct ClientList *pos;
-  struct PendingMessageList *pml;
-
-  while (NULL != (pos = nc->clients_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (nc->clients_head,
-                                 nc->clients_tail,
-                                 pos);
-    if (NULL != pos->th)
-    {
-      GNUNET_SERVER_notify_transmit_ready_cancel (pos->th);
-      pos->th = NULL;
-    }
-    GNUNET_SERVER_client_drop (pos->client);
-    while (NULL != (pml = pos->pending_head))
-    {
-      GNUNET_CONTAINER_DLL_remove (pos->pending_head,
-                                   pos->pending_tail,
-                                   pml);
-      GNUNET_free (pml);
-      pos->num_pending--;
-    }
-    GNUNET_assert (0 == pos->num_pending);
-    GNUNET_free (pos);
-  }
-  if (NULL != nc->server)
-    GNUNET_SERVER_disconnect_notify_cancel (nc->server,
-                                            &handle_client_disconnect,
-                                            nc);
-  GNUNET_free (nc);
-}
-
-
-/**
- * Add a client to the notification context.
- *
- * @param nc context to modify
- * @param client client to add
- */
-void
-GNUNET_SERVER_notification_context_add (struct GNUNET_SERVER_NotificationContext *nc,
-                                        struct GNUNET_SERVER_Client *client)
-{
-  struct ClientList *cl;
-
-  for (cl = nc->clients_head; NULL != cl; cl = cl->next)
-    if (cl->client == client)
-      return; /* already present */
-  cl = GNUNET_new (struct ClientList);
-  GNUNET_CONTAINER_DLL_insert (nc->clients_head,
-                               nc->clients_tail,
-                               cl);
-  cl->nc = nc;
-  cl->client = client;
-  GNUNET_SERVER_client_keep (client);
-}
-
-
-/**
- * Function called to notify a client about the socket begin ready to
- * queue more data.  @a buf will be NULL and @a size zero if the socket
- * was closed for writing in the meantime.
- *
- * @param cls the `struct ClientList *`
- * @param size number of bytes available in @a buf
- * @param buf where the callee should write the message
- * @return number of bytes written to buf
- */
-static size_t
-transmit_message (void *cls,
-                  size_t size,
-                  void *buf)
-{
-  struct ClientList *cl = cls;
-  char *cbuf = buf;
-  struct PendingMessageList *pml;
-  uint16_t msize;
-  size_t ret;
-
-  cl->th = NULL;
-  if (NULL == buf)
-  {
-    /* 'cl' should be freed via disconnect notification shortly */
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Failed to transmit message from NC queue to client\n");
-    return 0;
-  }
-  ret = 0;
-  while (NULL != (pml = cl->pending_head))
-  {
-    msize = ntohs (pml->msg->size);
-    if (size < msize)
-      break;
-    GNUNET_CONTAINER_DLL_remove (cl->pending_head,
-                                 cl->pending_tail,
-                                 pml);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Copying message of type %u and size %u from pending queue to transmission buffer\n",
-         ntohs (pml->msg->type),
-         msize);
-    GNUNET_memcpy (&cbuf[ret], pml->msg, msize);
-    ret += msize;
-    size -= msize;
-    GNUNET_free (pml);
-    cl->num_pending--;
-  }
-  if (NULL != pml)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Have %u messages left in NC queue, will try transmission again\n",
-         cl->num_pending);
-    cl->th =
-        GNUNET_SERVER_notify_transmit_ready (cl->client,
-                                             ntohs (pml->msg->size),
-                                             GNUNET_TIME_UNIT_FOREVER_REL,
-                                             &transmit_message, cl);
-  }
-  else
-  {
-    GNUNET_assert (0 == cl->num_pending);
-  }
-  return ret;
-}
-
-
-/**
- * Send a message to a particular client.
- *
- * @param nc context to modify
- * @param client client to transmit to
- * @param msg message to send
- * @param can_drop can this message be dropped due to queue length limitations
- */
-static void
-do_unicast (struct GNUNET_SERVER_NotificationContext *nc,
-            struct ClientList *client,
-            const struct GNUNET_MessageHeader *msg,
-            int can_drop)
-{
-  struct PendingMessageList *pml;
-  uint16_t size;
-
-  if ( (client->num_pending > nc->queue_length) &&
-       (GNUNET_YES == can_drop) )
-  {
-    LOG (GNUNET_ERROR_TYPE_INFO,
-         "Dropping message of type %u and size %u due to full queue (%u entries)\n",
-         ntohs (msg->type), ntohs (msg->size), (unsigned int) nc->queue_length);
-    return;                     /* drop! */
-  }
-  if (client->num_pending > nc->queue_length)
-  {
-    /* FIXME: consider checking for other messages in the
-     * queue that are 'droppable' */
-  }
-  client->num_pending++;
-  size = ntohs (msg->size);
-  pml = GNUNET_malloc (sizeof (struct PendingMessageList) + size);
-  pml->msg = (const struct GNUNET_MessageHeader *) &pml[1];
-  pml->can_drop = can_drop;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Adding message of type %u and size %u to pending queue (which has %u entries)\n",
-       ntohs (msg->type),
-       ntohs (msg->size),
-       (unsigned int) nc->queue_length);
-  GNUNET_memcpy (&pml[1], msg, size);
-  /* append */
-  GNUNET_CONTAINER_DLL_insert_tail (client->pending_head,
-                                    client->pending_tail,
-                                    pml);
-  if (NULL == client->th)
-    client->th =
-        GNUNET_SERVER_notify_transmit_ready (client->client,
-                                             ntohs (client->pending_head->
-                                                    msg->size),
-                                             GNUNET_TIME_UNIT_FOREVER_REL,
-                                             &transmit_message, client);
-}
-
-
-/**
- * Send a message to a particular client; must have
- * already been added to the notification context.
- *
- * @param nc context to modify
- * @param client client to transmit to
- * @param msg message to send
- * @param can_drop can this message be dropped due to queue length limitations
- */
-void
-GNUNET_SERVER_notification_context_unicast (struct GNUNET_SERVER_NotificationContext *nc,
-                                            struct GNUNET_SERVER_Client *client,
-                                            const struct GNUNET_MessageHeader *msg,
-                                            int can_drop)
-{
-  struct ClientList *pos;
-
-  for (pos = nc->clients_head; NULL != pos; pos = pos->next)
-    if (pos->client == client)
-      break;
-  GNUNET_assert (NULL != pos);
-  do_unicast (nc, pos, msg, can_drop);
-}
-
-
-/**
- * Send a message to all clients of this context.
- *
- * @param nc context to modify
- * @param msg message to send
- * @param can_drop can this message be dropped due to queue length limitations
- */
-void
-GNUNET_SERVER_notification_context_broadcast (struct
-                                              GNUNET_SERVER_NotificationContext *nc,
-                                              const struct GNUNET_MessageHeader *msg,
-                                              int can_drop)
-{
-  struct ClientList *pos;
-
-  for (pos = nc->clients_head; NULL != pos; pos = pos->next)
-    do_unicast (nc, pos, msg, can_drop);
-}
-
-
-/**
- * Return active number of subscribers in this context.
- *
- * @param nc context to query
- * @return number of current subscribers
- */
-unsigned int
-GNUNET_SERVER_notification_context_get_size (struct GNUNET_SERVER_NotificationContext *nc)
-{
-  unsigned int num;
-  struct ClientList *pos;
-
-  num = 0;
-  for (pos = nc->clients_head; NULL != pos; pos = pos->next)
-    num++;
-  return num;
-}
-
-/* end of server_nc.c */
diff --git a/src/util/server_tc.c b/src/util/server_tc.c
deleted file mode 100644
index 8ae380a85..000000000
--- a/src/util/server_tc.c
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file util/server_tc.c
- * @brief convenience functions for transmission of
- *        complex responses as a server
- * @author Christian Grothoff
- */
-
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-
-#define LOG(kind,...) GNUNET_log_from (kind, "util-server-tc", __VA_ARGS__)
-
-
-/**
- * How much buffer space do we want to have at least
- * before transmitting another increment?
- */
-#define MIN_BLOCK_SIZE 128
-
-
-
-struct GNUNET_SERVER_TransmitContext
-{
-  /**
-   * Which client are we transmitting to?
-   */
-  struct GNUNET_SERVER_Client *client;
-
-  /**
-   * Transmission buffer. (current offset for writing).
-   */
-  char *buf;
-
-  /**
-   * Number of bytes in buf.
-   */
-  size_t total;
-
-  /**
-   * Offset for writing in buf.
-   */
-  size_t off;
-
-  /**
-   * Timeout for this request.
-   */
-  struct GNUNET_TIME_Absolute timeout;
-};
-
-
-/**
- * Helper function for incremental transmission of the response.
- */
-static size_t
-transmit_response (void *cls, size_t size, void *buf)
-{
-  struct GNUNET_SERVER_TransmitContext *tc = cls;
-  size_t msize;
-
-  if (NULL == buf)
-  {
-    GNUNET_SERVER_transmit_context_destroy (tc, GNUNET_SYSERR);
-    return 0;
-  }
-  if (tc->total - tc->off > size)
-    msize = size;
-  else
-    msize = tc->total - tc->off;
-  GNUNET_memcpy (buf, &tc->buf[tc->off], msize);
-  tc->off += msize;
-  if (tc->total == tc->off)
-  {
-    GNUNET_SERVER_receive_done (tc->client, GNUNET_OK);
-    GNUNET_SERVER_client_drop (tc->client);
-    GNUNET_free_non_null (tc->buf);
-    GNUNET_free (tc);
-  }
-  else
-  {
-    if (NULL ==
-        GNUNET_SERVER_notify_transmit_ready (tc->client,
-                                             GNUNET_MIN (MIN_BLOCK_SIZE,
-                                                         tc->total - tc->off),
-                                             GNUNET_TIME_absolute_get_remaining
-                                             (tc->timeout), &transmit_response,
-                                             tc))
-    {
-      GNUNET_break (0);
-      GNUNET_SERVER_transmit_context_destroy (tc, GNUNET_SYSERR);
-    }
-  }
-  return msize;
-}
-
-
-/**
- * Create a new transmission context for the
- * given client.
- *
- * @param client client to create the context for.
- * @return NULL on error
- */
-struct GNUNET_SERVER_TransmitContext *
-GNUNET_SERVER_transmit_context_create (struct GNUNET_SERVER_Client *client)
-{
-  struct GNUNET_SERVER_TransmitContext *tc;
-
-  GNUNET_assert (NULL != client);
-  tc = GNUNET_new (struct GNUNET_SERVER_TransmitContext);
-  GNUNET_SERVER_client_keep (client);
-  tc->client = client;
-  return tc;
-}
-
-
-/**
- * Append a message to the transmission context.
- * All messages in the context will be sent by
- * the transmit_context_run method.
- *
- * @param tc context to use
- * @param data what to append to the result message
- * @param length length of data
- * @param type type of the message
- */
-void
-GNUNET_SERVER_transmit_context_append_data (struct GNUNET_SERVER_TransmitContext
-                                            *tc, const void *data,
-                                            size_t length, uint16_t type)
-{
-  struct GNUNET_MessageHeader *msg;
-  size_t size;
-
-  GNUNET_assert (length < GNUNET_SERVER_MAX_MESSAGE_SIZE);
-  size = length + sizeof (struct GNUNET_MessageHeader);
-  GNUNET_assert (size > length);
-  tc->buf = GNUNET_realloc (tc->buf, tc->total + size);
-  msg = (struct GNUNET_MessageHeader *) &tc->buf[tc->total];
-  tc->total += size;
-  msg->size = htons (size);
-  msg->type = htons (type);
-  GNUNET_memcpy (&msg[1], data, length);
-}
-
-
-/**
- * Append a message to the transmission context.
- * All messages in the context will be sent by
- * the transmit_context_run method.
- *
- * @param tc context to use
- * @param msg message to append
- */
-void
-GNUNET_SERVER_transmit_context_append_message (struct
-                                               GNUNET_SERVER_TransmitContext
-                                               *tc,
-                                               const struct GNUNET_MessageHeader
-                                               *msg)
-{
-  struct GNUNET_MessageHeader *m;
-  uint16_t size;
-
-  size = ntohs (msg->size);
-  tc->buf = GNUNET_realloc (tc->buf, tc->total + size);
-  m = (struct GNUNET_MessageHeader *) &tc->buf[tc->total];
-  tc->total += size;
-  GNUNET_memcpy (m, msg, size);
-}
-
-
-/**
- * Execute a transmission context.  If there is
- * an error in the transmission, the #GNUNET_SERVER_receive_done()
- * method will be called with an error code (#GNUNET_SYSERR),
- * otherwise with #GNUNET_OK.
- *
- * @param tc transmission context to use
- * @param timeout when to time out and abort the transmission
- */
-void
-GNUNET_SERVER_transmit_context_run (struct GNUNET_SERVER_TransmitContext *tc,
-                                    struct GNUNET_TIME_Relative timeout)
-{
-  tc->timeout = GNUNET_TIME_relative_to_absolute (timeout);
-  if (NULL ==
-      GNUNET_SERVER_notify_transmit_ready (tc->client,
-                                           GNUNET_MIN (MIN_BLOCK_SIZE,
-                                                       tc->total), timeout,
-                                           &transmit_response, tc))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_transmit_context_destroy (tc, GNUNET_SYSERR);
-  }
-}
-
-
-/**
- * Destroy a transmission context. This function must not be called
- * after 'GNUNET_SERVER_transmit_context_run'.
- *
- * @param tc transmission context to destroy
- * @param success code to give to 'GNUNET_SERVER_receive_done' for
- *        the client:  GNUNET_OK to keep the connection open and
- *                          continue to receive
- *                GNUNET_NO to close the connection (normal behavior)
- *                GNUNET_SYSERR to close the connection (signal
- *                          serious error)
- */
-void
-GNUNET_SERVER_transmit_context_destroy (struct GNUNET_SERVER_TransmitContext
-                                        *tc, int success)
-{
-  GNUNET_SERVER_receive_done (tc->client, success);
-  GNUNET_SERVER_client_drop (tc->client);
-  GNUNET_free_non_null (tc->buf);
-  GNUNET_free (tc);
-}
-
-
-/* end of server_tc.c */
diff --git a/src/util/service.c b/src/util/service.c
index 496904fb1..800d09a42 100644
--- a/src/util/service.c
+++ b/src/util/service.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2009, 2012 GNUnet e.V.
+     Copyright (C) 2016 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -19,9 +19,10 @@
 */
 
 /**
- * @file util/service.c
- * @brief functions related to starting services
+ * @file util/service_new.c
+ * @brief functions related to starting services (redesign)
  * @author Christian Grothoff
+ * @author Florian Dold
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
@@ -43,77 +44,44 @@
 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-service", syscall, filename)
 
 
-/* ******************* access control ******************** */
-
 /**
- * Check if the given IP address is in the list of IP addresses.
- *
- * @param list a list of networks
- * @param add the IP to check (in network byte order)
- * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
+ * Information the service tracks per listen operation.
  */
-static int
-check_ipv4_listed (const struct GNUNET_STRINGS_IPv4NetworkPolicy *list,
-                   const struct in_addr *add)
+struct ServiceListenContext
 {
-  unsigned int i;
 
-  if (NULL == list)
-    return GNUNET_NO;
-  i = 0;
-  while ((list[i].network.s_addr != 0) || (list[i].netmask.s_addr != 0))
-  {
-    if ((add->s_addr & list[i].netmask.s_addr) ==
-        (list[i].network.s_addr & list[i].netmask.s_addr))
-      return GNUNET_YES;
-    i++;
-  }
-  return GNUNET_NO;
-}
+  /**
+   * Kept in a DLL.
+   */
+  struct ServiceListenContext *next;
 
+  /**
+   * Kept in a DLL.
+   */
+  struct ServiceListenContext *prev;
 
-/**
- * Check if the given IP address is in the list of IP addresses.
- *
- * @param list a list of networks
- * @param ip the IP to check (in network byte order)
- * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
- */
-static int
-check_ipv6_listed (const struct GNUNET_STRINGS_IPv6NetworkPolicy *list,
-                   const struct in6_addr *ip)
-{
-  unsigned int i;
-  unsigned int j;
-  struct in6_addr zero;
+  /**
+   * Service this listen context belongs to.
+   */
+  struct GNUNET_SERVICE_Handle *sh;
 
-  if (NULL == list)
-    return GNUNET_NO;
-  memset (&zero, 0, sizeof (struct in6_addr));
-  i = 0;
-NEXT:
-  while (0 != memcmp (&zero, &list[i].network, sizeof (struct in6_addr)))
-  {
-    for (j = 0; j < sizeof (struct in6_addr) / sizeof (int); j++)
-      if (((((int *) ip)[j] & ((int *) &list[i].netmask)[j])) !=
-          (((int *) &list[i].network)[j] & ((int *) &list[i].netmask)[j]))
-      {
-        i++;
-        goto NEXT;
-      }
-    return GNUNET_YES;
-  }
-  return GNUNET_NO;
-}
+  /**
+   * Socket we are listening on.
+   */
+  struct GNUNET_NETWORK_Handle *listen_socket;
 
+  /**
+   * Task scheduled to do the listening.
+   */
+  struct GNUNET_SCHEDULER_Task *listen_task;
 
-/* ****************** service struct ****************** */
+};
 
 
 /**
- * Context for "service_task".
+ * Handle to a service.
  */
-struct GNUNET_SERVICE_Context
+struct GNUNET_SERVICE_Handle
 {
   /**
    * Our configuration.
@@ -121,25 +89,54 @@ struct GNUNET_SERVICE_Context
   const struct GNUNET_CONFIGURATION_Handle *cfg;
 
   /**
-   * Handle for the server.
+   * Name of our service.
    */
-  struct GNUNET_SERVER_Handle *server;
+  const char *service_name;
 
   /**
-   * NULL-terminated array of addresses to bind to, NULL if we got pre-bound
-   * listen sockets.
+   * Main service-specific task to run.
    */
-  struct sockaddr **addrs;
+  GNUNET_SERVICE_InitCallback service_init_cb;
 
   /**
-   * Name of our service.
+   * Function to call when clients connect.
    */
-  const char *service_name;
+  GNUNET_SERVICE_ConnectHandler connect_cb;
 
   /**
-   * Main service-specific task to run.
+   * Function to call when clients disconnect / are disconnected.
    */
-  GNUNET_SERVICE_Main task;
+  GNUNET_SERVICE_DisconnectHandler disconnect_cb;
+
+  /**
+   * Closure for @e service_init_cb, @e connect_cb, @e disconnect_cb.
+   */
+  void *cb_cls;
+
+  /**
+   * DLL of listen sockets used to accept new connections.
+   */
+  struct ServiceListenContext *slc_head;
+
+  /**
+   * DLL of listen sockets used to accept new connections.
+   */
+  struct ServiceListenContext *slc_tail;
+
+  /**
+   * Our clients, kept in a DLL.
+   */
+  struct GNUNET_SERVICE_Client *clients_head;
+
+  /**
+   * Our clients, kept in a DLL.
+   */
+  struct GNUNET_SERVICE_Client *clients_tail;
+
+  /**
+   * Message handlers to use for all clients.
+   */
+  struct GNUNET_MQ_MessageHandler *handlers;
 
   /**
    * Closure for @e task.
@@ -169,30 +166,38 @@ struct GNUNET_SERVICE_Context
   struct GNUNET_STRINGS_IPv6NetworkPolicy *v6_allowed;
 
   /**
-   * My (default) message handlers.  Adjusted copy
-   * of "defhandlers".
+   * Do we require a matching UID for UNIX domain socket connections?
+   * #GNUNET_NO means that the UID does not have to match (however,
+   * @e match_gid may still impose other access control checks).
    */
-  struct GNUNET_SERVER_MessageHandler *my_handlers;
+  int match_uid;
 
   /**
-   * Array of the lengths of the entries in addrs.
+   * Do we require a matching GID for UNIX domain socket connections?
+   * Ignored if @e match_uid is #GNUNET_YES.  Note that this is about
+   * checking that the client's UID is in our group OR that the
+   * client's GID is our GID.  If both "match_gid" and @e match_uid are
+   * #GNUNET_NO, all users on the local system have access.
    */
-  socklen_t *addrlens;
+  int match_gid;
 
   /**
-   * NULL-terminated array of listen sockets we should take over.
+   * Set to #GNUNET_YES if we got a shutdown signal and terminate
+   * the service if #have_non_monitor_clients() returns #GNUNET_YES.
    */
-  struct GNUNET_NETWORK_Handle **lsocks;
+  int got_shutdown;
 
   /**
-   * Task ID of the shutdown task.
+   * Our options.
    */
-  struct GNUNET_SCHEDULER_Task *shutdown_task;
+  enum GNUNET_SERVICE_Options options;
 
   /**
-   * Idle timeout for server.
+   * If we are daemonizing, this FD is set to the
+   * pipe to the parent.  Send '.' if we started
+   * ok, '!' if not.  -1 if we are not daemonizing.
    */
-  struct GNUNET_TIME_Relative timeout;
+  int ready_confirm_fd;
 
   /**
    * Overall success/failure of the service start.
@@ -200,182 +205,203 @@ struct GNUNET_SERVICE_Context
   int ret;
 
   /**
-   * If we are daemonizing, this FD is set to the
-   * pipe to the parent.  Send '.' if we started
-   * ok, '!' if not.  -1 if we are not daemonizing.
+   * If #GNUNET_YES, consider unknown message types an error where the
+   * client is disconnected.
    */
-  int ready_confirm_fd;
+  int require_found;
+};
+
+
+/**
+ * Handle to a client that is connected to a service.
+ */
+struct GNUNET_SERVICE_Client
+{
 
   /**
-   * Do we close connections if we receive messages
-   * for which we have no handler?
+   * Kept in a DLL.
    */
-  int require_found;
+  struct GNUNET_SERVICE_Client *next;
 
   /**
-   * Do we require a matching UID for UNIX domain socket connections?
-   * #GNUNET_NO means that the UID does not have to match (however,
-   * @e match_gid may still impose other access control checks).
+   * Kept in a DLL.
    */
-  int match_uid;
+  struct GNUNET_SERVICE_Client *prev;
 
   /**
-   * Do we require a matching GID for UNIX domain socket connections?
-   * Ignored if @e match_uid is #GNUNET_YES.  Note that this is about
-   * checking that the client's UID is in our group OR that the
-   * client's GID is our GID.  If both "match_gid" and @e match_uid are
-   * #GNUNET_NO, all users on the local system have access.
+   * Service that this client belongs to.
    */
-  int match_gid;
+  struct GNUNET_SERVICE_Handle *sh;
 
   /**
-   * Our options.
+   * Socket of this client.
    */
-  enum GNUNET_SERVICE_Options options;
+  struct GNUNET_NETWORK_Handle *sock;
 
-};
+  /**
+   * Message queue for the client.
+   */
+  struct GNUNET_MQ_Handle *mq;
+
+  /**
+   * Tokenizer we use for processing incoming data.
+   */
+  struct GNUNET_MessageStreamTokenizer *mst;
+
+  /**
+   * Task that warns about missing calls to
+   * #GNUNET_SERVICE_client_continue().
+   */
+  struct GNUNET_SCHEDULER_Task *warn_task;
 
+  /**
+   * Task run to finish dropping the client after the stack has
+   * properly unwound.
+   */
+  struct GNUNET_SCHEDULER_Task *drop_task;
+
+  /**
+   * Task that receives data from the client to
+   * pass it to the handlers.
+   */
+  struct GNUNET_SCHEDULER_Task *recv_task;
+
+  /**
+   * Task that transmit data to the client.
+   */
+  struct GNUNET_SCHEDULER_Task *send_task;
+
+  /**
+   * Pointer to the message to be transmitted by @e send_task.
+   */
+  const struct GNUNET_MessageHeader *msg;
+
+  /**
+   * User context value, value returned from
+   * the connect callback.
+   */
+  void *user_context;
+
+  /**
+   * Time when we last gave a message from this client
+   * to the application.
+   */
+  struct GNUNET_TIME_Absolute warn_start;
+
+  /**
+   * Current position in @e msg at which we are transmitting.
+   */
+  size_t msg_pos;
+
+  /**
+   * Persist the file handle for this client no matter what happens,
+   * force the OS to close once the process actually dies.  Should only
+   * be used in special cases!
+   */
+  int persist;
+
+  /**
+   * Is this client a 'monitor' client that should not be counted
+   * when deciding on destroying the server during soft shutdown?
+   * (see also #GNUNET_SERVICE_start)
+   */
+  int is_monitor;
+
+  /**
+   * Are we waiting for the application to call #GNUNET_SERVICE_client_continue()?
+   */
+  int needs_continue;
+
+  /**
+   * Type of last message processed (for warn_no_receive_done).
+   */
+  uint16_t warn_type;
+};
 
-/* ****************** message handlers ****************** */
 
 /**
- * Send a 'TEST' message back to the client.
+ * Check if any of the clients we have left are unrelated to
+ * monitoring.
  *
- * @param cls the 'struct GNUNET_SERVER_Client' to send TEST to
- * @param size number of bytes available in 'buf'
- * @param buf where to copy the message
- * @return number of bytes written to 'buf'
+ * @param sh service to check clients for
+ * @return #GNUNET_YES if we have non-monitoring clients left
  */
-static size_t
-write_test (void *cls, size_t size, void *buf)
+static int
+have_non_monitor_clients (struct GNUNET_SERVICE_Handle *sh)
 {
-  struct GNUNET_SERVER_Client *client = cls;
-  struct GNUNET_MessageHeader *msg;
+  struct GNUNET_SERVICE_Client *client;
 
-  if (size < sizeof (struct GNUNET_MessageHeader))
+  for (client = sh->clients_head;NULL != client; client = client->next)
   {
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return 0;                   /* client disconnected */
+    if (client->is_monitor)
+      continue;
+    return GNUNET_YES;
   }
-  msg = (struct GNUNET_MessageHeader *) buf;
-  msg->type = htons (GNUNET_MESSAGE_TYPE_TEST);
-  msg->size = htons (sizeof (struct GNUNET_MessageHeader));
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-  return sizeof (struct GNUNET_MessageHeader);
+  return GNUNET_NO;
 }
 
 
 /**
- * Handler for TEST message.
+ * Shutdown task triggered when a service should be terminated.
+ * This considers active clients and the service options to see
+ * how this specific service is to be terminated, and depending
+ * on this proceeds with the shutdown logic.
  *
- * @param cls closure (refers to service)
- * @param client identification of the client
- * @param message the actual message
+ * @param cls our `struct GNUNET_SERVICE_Handle`
  */
 static void
-handle_test (void *cls, struct GNUNET_SERVER_Client *client,
-             const struct GNUNET_MessageHeader *message)
+service_shutdown (void *cls)
 {
-  /* simply bounce message back to acknowledge */
-  if (NULL ==
-      GNUNET_SERVER_notify_transmit_ready (client,
-                                           sizeof (struct GNUNET_MessageHeader),
-                                           GNUNET_TIME_UNIT_FOREVER_REL,
-                                           &write_test, client))
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-}
-
-
-/**
- * Default handlers for all services.  Will be copied and the
- * "callback_cls" fields will be replaced with the specific service
- * struct.
- */
-static const struct GNUNET_SERVER_MessageHandler defhandlers[] = {
-  {&handle_test, NULL, GNUNET_MESSAGE_TYPE_TEST,
-   sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-/* ****************** service core routines ************** */
+  struct GNUNET_SERVICE_Handle *sh = cls;
 
-
-/**
- * Check if access to the service is allowed from the given address.
- *
- * @param cls closure
- * @param uc credentials, if available, otherwise NULL
- * @param addr address
- * @param addrlen length of address
- * @return #GNUNET_YES to allow, #GNUNET_NO to deny, #GNUNET_SYSERR
- *   for unknown address family (will be denied).
- */
-static int
-check_access (void *cls, const struct GNUNET_CONNECTION_Credentials *uc,
-              const struct sockaddr *addr, socklen_t addrlen)
-{
-  struct GNUNET_SERVICE_Context *sctx = cls;
-  const struct sockaddr_in *i4;
-  const struct sockaddr_in6 *i6;
-  int ret;
-
-  switch (addr->sa_family)
+  switch (sh->options)
   {
-  case AF_INET:
-    GNUNET_assert (addrlen == sizeof (struct sockaddr_in));
-    i4 = (const struct sockaddr_in *) addr;
-    ret = ((NULL == sctx->v4_allowed) ||
-           (check_ipv4_listed (sctx->v4_allowed, &i4->sin_addr))) &&
-        ((NULL == sctx->v4_denied) ||
-         (!check_ipv4_listed (sctx->v4_denied, &i4->sin_addr)));
+  case GNUNET_SERVICE_OPTION_NONE:
+    GNUNET_SERVICE_shutdown (sh);
     break;
-  case AF_INET6:
-    GNUNET_assert (addrlen == sizeof (struct sockaddr_in6));
-    i6 = (const struct sockaddr_in6 *) addr;
-    ret = ((NULL == sctx->v6_allowed) ||
-           (check_ipv6_listed (sctx->v6_allowed, &i6->sin6_addr))) &&
-        ((NULL == sctx->v6_denied) ||
-         (!check_ipv6_listed (sctx->v6_denied, &i6->sin6_addr)));
+  case GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN:
+    /* This task should never be run if we are using
+       the manual shutdown. */
+    GNUNET_assert (0);
     break;
-#ifndef WINDOWS
-  case AF_UNIX:
-    ret = GNUNET_OK;            /* controlled using file-system ACL now */
+  case GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN:
+    sh->got_shutdown = GNUNET_YES;
+    GNUNET_SERVICE_suspend (sh);
+    if (GNUNET_NO == have_non_monitor_clients (sh))
+      GNUNET_SERVICE_shutdown (sh);
     break;
-#endif
-  default:
-    LOG (GNUNET_ERROR_TYPE_WARNING, _("Unknown address family %d\n"),
-         addr->sa_family);
-    return GNUNET_SYSERR;
-  }
-  if (GNUNET_OK != ret)
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         _("Access from `%s' denied to service `%s'\n"),
-         GNUNET_a2s (addr, addrlen),
-         sctx->service_name);
   }
-  return ret;
 }
 
 
 /**
- * Get the name of the file where we will
- * write the PID of the service.
+ * First task run by any service.  Initializes our shutdown task,
+ * starts the listening operation on our listen sockets and launches
+ * the custom logic of the application service.
  *
- * @param sctx service context
- * @return name of the file for the process ID
+ * @param cls our `struct GNUNET_SERVICE_Handle`
  */
-static char *
-get_pid_file_name (struct GNUNET_SERVICE_Context *sctx)
+static void
+service_main (void *cls)
 {
-  char *pif;
+  struct GNUNET_SERVICE_Handle *sh = cls;
 
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->service_name,
-                                               "PIDFILE", &pif))
-    return NULL;
-  return pif;
+  if (GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN != sh->options)
+    GNUNET_SCHEDULER_add_shutdown (&service_shutdown,
+                                   sh);
+  GNUNET_SERVICE_resume (sh);
+
+  if (-1 != sh->ready_confirm_fd)
+  {
+    GNUNET_break (1 == WRITE (sh->ready_confirm_fd, ".", 1));
+    GNUNET_break (0 == CLOSE (sh->ready_confirm_fd));
+    sh->ready_confirm_fd = -1;
+  }
+
+  if (NULL != sh->service_init_cb)
+    sh->service_init_cb (sh->cb_cls,
+                         sh->cfg,
+                         sh);
 }
 
 
@@ -383,32 +409,37 @@ get_pid_file_name (struct GNUNET_SERVICE_Context *sctx)
  * Parse an IPv4 access control list.
  *
  * @param ret location where to write the ACL (set)
- * @param sctx service context to use to get the configuration
+ * @param sh service context to use to get the configuration
  * @param option name of the ACL option to parse
  * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
  *         no ACL configured)
  */
 static int
 process_acl4 (struct GNUNET_STRINGS_IPv4NetworkPolicy **ret,
-              struct GNUNET_SERVICE_Context *sctx,
+              struct GNUNET_SERVICE_Handle *sh,
               const char *option)
 {
   char *opt;
 
-  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, option))
+  if (! GNUNET_CONFIGURATION_have_value (sh->cfg,
+                                         sh->service_name,
+                                         option))
   {
     *ret = NULL;
     return GNUNET_OK;
   }
   GNUNET_break (GNUNET_OK ==
-                GNUNET_CONFIGURATION_get_value_string (sctx->cfg,
-                                                       sctx->service_name,
-                                                       option, &opt));
+                GNUNET_CONFIGURATION_get_value_string (sh->cfg,
+                                                       sh->service_name,
+                                                       option,
+                                                       &opt));
   if (NULL == (*ret = GNUNET_STRINGS_parse_ipv4_policy (opt)))
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
          _("Could not parse IPv4 network specification `%s' for `%s:%s'\n"),
-         opt, sctx->service_name, option);
+         opt,
+         sh->service_name,
+         option);
     GNUNET_free (opt);
     return GNUNET_SYSERR;
   }
@@ -421,32 +452,37 @@ process_acl4 (struct GNUNET_STRINGS_IPv4NetworkPolicy **ret,
  * Parse an IPv6 access control list.
  *
  * @param ret location where to write the ACL (set)
- * @param sctx service context to use to get the configuration
+ * @param sh service context to use to get the configuration
  * @param option name of the ACL option to parse
  * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
  *         no ACL configured)
  */
 static int
 process_acl6 (struct GNUNET_STRINGS_IPv6NetworkPolicy **ret,
-              struct GNUNET_SERVICE_Context *sctx,
+              struct GNUNET_SERVICE_Handle *sh,
               const char *option)
 {
   char *opt;
 
-  if (!GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, option))
+  if (! GNUNET_CONFIGURATION_have_value (sh->cfg,
+                                         sh->service_name,
+                                         option))
   {
     *ret = NULL;
     return GNUNET_OK;
   }
   GNUNET_break (GNUNET_OK ==
-                GNUNET_CONFIGURATION_get_value_string (sctx->cfg,
-                                                       sctx->service_name,
-                                                       option, &opt));
+                GNUNET_CONFIGURATION_get_value_string (sh->cfg,
+                                                       sh->service_name,
+                                                       option,
+                                                       &opt));
   if (NULL == (*ret = GNUNET_STRINGS_parse_ipv6_policy (opt)))
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
          _("Could not parse IPv6 network specification `%s' for `%s:%s'\n"),
-         opt, sctx->service_name, option);
+         opt,
+         sh->service_name,
+         option);
     GNUNET_free (opt);
     return GNUNET_SYSERR;
   }
@@ -476,7 +512,9 @@ add_unixpath (struct sockaddr **saddrs,
 
   un = GNUNET_new (struct sockaddr_un);
   un->sun_family = AF_UNIX;
-  strncpy (un->sun_path, unixpath, sizeof (un->sun_path) - 1);
+  strncpy (un->sun_path,
+           unixpath,
+           sizeof (un->sun_path) - 1);
 #ifdef LINUX
   if (GNUNET_YES == abstract)
     un->sun_path[0] = '\0';
@@ -514,11 +552,11 @@ add_unixpath (struct sockaddr **saddrs,
  *              zero (in this case, `*addrs` and `*addr_lens` will be
  *              set to NULL).
  */
-int
-GNUNET_SERVICE_get_server_addresses (const char *service_name,
-                                     const struct GNUNET_CONFIGURATION_Handle *cfg,
-                                     struct sockaddr ***addrs,
-                                     socklen_t ** addr_lens)
+static int
+get_server_addresses (const char *service_name,
+                      const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      struct sockaddr ***addrs,
+                      socklen_t **addr_lens)
 {
   int disablev6;
   struct GNUNET_NETWORK_Handle *desc;
@@ -539,11 +577,15 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
   *addrs = NULL;
   *addr_lens = NULL;
   desc = NULL;
-  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "DISABLEV6"))
+  if (GNUNET_CONFIGURATION_have_value (cfg,
+                                       service_name,
+                                       "DISABLEV6"))
   {
     if (GNUNET_SYSERR ==
         (disablev6 =
-         GNUNET_CONFIGURATION_get_value_yesno (cfg, service_name, "DISABLEV6")))
+         GNUNET_CONFIGURATION_get_value_yesno (cfg,
+                                               service_name,
+                                               "DISABLEV6")))
       return GNUNET_SYSERR;
   }
   else
@@ -552,33 +594,44 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
   if (! disablev6)
   {
     /* probe IPv6 support */
-    desc = GNUNET_NETWORK_socket_create (PF_INET6, SOCK_STREAM, 0);
+    desc = GNUNET_NETWORK_socket_create (PF_INET6,
+                                         SOCK_STREAM,
+                                         0);
     if (NULL == desc)
     {
-      if ((ENOBUFS == errno) || (ENOMEM == errno) || (ENFILE == errno) ||
-          (EACCES == errno))
+      if ( (ENOBUFS == errno) ||
+           (ENOMEM == errno) ||
+           (ENFILE == errno) ||
+           (EACCES == errno) )
       {
-        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "socket");
+        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                      "socket");
         return GNUNET_SYSERR;
       }
       LOG (GNUNET_ERROR_TYPE_INFO,
            _("Disabling IPv6 support for service `%s', failed to create IPv6 socket: %s\n"),
-           service_name, STRERROR (errno));
+           service_name,
+           STRERROR (errno));
       disablev6 = GNUNET_YES;
     }
     else
     {
-      GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (desc));
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_NETWORK_socket_close (desc));
       desc = NULL;
     }
   }
 
   port = 0;
-  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "PORT"))
+  if (GNUNET_CONFIGURATION_have_value (cfg,
+                                       service_name,
+                                       "PORT"))
   {
     if (GNUNET_OK !=
-        GNUNET_CONFIGURATION_get_value_number (cfg, service_name,
-                                               "PORT", &port))
+        GNUNET_CONFIGURATION_get_value_number (cfg,
+                                               service_name,
+                                               "PORT",
+                                               &port))
     {
       LOG (GNUNET_ERROR_TYPE_ERROR,
            _("Require valid port number for service `%s' in configuration!\n"),
@@ -593,11 +646,15 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     }
   }
 
-  if (GNUNET_CONFIGURATION_have_value (cfg, service_name, "BINDTO"))
+  if (GNUNET_CONFIGURATION_have_value (cfg,
+                                       service_name,
+                                       "BINDTO"))
   {
     GNUNET_break (GNUNET_OK ==
-                  GNUNET_CONFIGURATION_get_value_string (cfg, service_name,
-                                                         "BINDTO", &hostname));
+                  GNUNET_CONFIGURATION_get_value_string (cfg,
+                                                         service_name,
+                                                         "BINDTO",
+                                                         &hostname));
   }
   else
     hostname = NULL;
@@ -606,10 +663,14 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
   abstract = GNUNET_NO;
 #ifdef AF_UNIX
   if ((GNUNET_YES ==
-       GNUNET_CONFIGURATION_have_value (cfg, service_name, "UNIXPATH")) &&
+       GNUNET_CONFIGURATION_have_value (cfg,
+                                        service_name,
+                                        "UNIXPATH")) &&
       (GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_filename (cfg, service_name, "UNIXPATH",
-                                              &unixpath)) &&
+       GNUNET_CONFIGURATION_get_value_filename (cfg,
+                                                service_name,
+                                                "UNIXPATH",
+                                                &unixpath)) &&
       (0 < strlen (unixpath)))
   {
     /* probe UNIX support */
@@ -618,7 +679,8 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     if (strlen (unixpath) >= sizeof (s_un.sun_path))
     {
       LOG (GNUNET_ERROR_TYPE_WARNING,
-           _("UNIXPATH `%s' too long, maximum length is %llu\n"), unixpath,
+           _("UNIXPATH `%s' too long, maximum length is %llu\n"),
+           unixpath,
            (unsigned long long) sizeof (s_un.sun_path));
       unixpath = GNUNET_NETWORK_shorten_unixpath (unixpath);
       LOG (GNUNET_ERROR_TYPE_INFO,
@@ -632,22 +694,27 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     if (GNUNET_SYSERR == abstract)
       abstract = GNUNET_NO;
 #endif
-    if ((GNUNET_YES != abstract)
-        && (GNUNET_OK !=
-            GNUNET_DISK_directory_create_for_file (unixpath)))
+    if ( (GNUNET_YES != abstract) &&
+         (GNUNET_OK !=
+          GNUNET_DISK_directory_create_for_file (unixpath)) )
       GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
                                 "mkdir",
                                 unixpath);
   }
   if (NULL != unixpath)
   {
-    desc = GNUNET_NETWORK_socket_create (AF_UNIX, SOCK_STREAM, 0);
+    desc = GNUNET_NETWORK_socket_create (AF_UNIX,
+                                         SOCK_STREAM,
+                                         0);
     if (NULL == desc)
     {
-      if ((ENOBUFS == errno) || (ENOMEM == errno) || (ENFILE == errno) ||
+      if ((ENOBUFS == errno) ||
+          (ENOMEM == errno) ||
+          (ENFILE == errno) ||
           (EACCES == errno))
       {
-        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "socket");
+        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                      "socket");
         GNUNET_free_non_null (hostname);
         GNUNET_free (unixpath);
         return GNUNET_SYSERR;
@@ -661,7 +728,8 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     }
     else
     {
-      GNUNET_break (GNUNET_OK == GNUNET_NETWORK_socket_close (desc));
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_NETWORK_socket_close (desc));
       desc = NULL;
     }
   }
@@ -677,9 +745,14 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
   }
   if (0 == port)
   {
-    saddrs = GNUNET_malloc (2 * sizeof (struct sockaddr *));
-    saddrlens = GNUNET_malloc (2 * sizeof (socklen_t));
-    add_unixpath (saddrs, saddrlens, unixpath, abstract);
+    saddrs = GNUNET_new_array (2,
+                               struct sockaddr *);
+    saddrlens = GNUNET_new_array (2,
+                                  socklen_t);
+    add_unixpath (saddrs,
+                  saddrlens,
+                  unixpath,
+                  abstract);
     GNUNET_free_non_null (unixpath);
     GNUNET_free_non_null (hostname);
     *addrs = saddrs;
@@ -693,11 +766,16 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
          "Resolving `%s' since that is where `%s' will bind to.\n",
          hostname,
          service_name);
-    memset (&hints, 0, sizeof (struct addrinfo));
+    memset (&hints,
+            0,
+            sizeof (struct addrinfo));
     if (disablev6)
       hints.ai_family = AF_INET;
     hints.ai_protocol = IPPROTO_TCP;
-    if ((0 != (ret = getaddrinfo (hostname, NULL, &hints, &res))) ||
+    if ((0 != (ret = getaddrinfo (hostname,
+                                  NULL,
+                                  &hints,
+                                  &res))) ||
         (NULL == res))
     {
       LOG (GNUNET_ERROR_TYPE_ERROR,
@@ -713,7 +791,8 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     while (NULL != (pos = next))
     {
       next = pos->ai_next;
-      if ((disablev6) && (pos->ai_family == AF_INET6))
+      if ( (disablev6) &&
+           (pos->ai_family == AF_INET6) )
         continue;
       i++;
     }
@@ -731,32 +810,45 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
     resi = i;
     if (NULL != unixpath)
       resi++;
-    saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
-    saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+    saddrs = GNUNET_new_array (resi + 1,
+                               struct sockaddr *);
+    saddrlens = GNUNET_new_array (resi + 1,
+                                  socklen_t);
     i = 0;
     if (NULL != unixpath)
     {
-      add_unixpath (saddrs, saddrlens, unixpath, abstract);
+      add_unixpath (saddrs,
+                    saddrlens,
+                    unixpath,
+                    abstract);
       i++;
     }
     next = res;
     while (NULL != (pos = next))
     {
       next = pos->ai_next;
-      if ((disablev6) && (AF_INET6 == pos->ai_family))
+      if ( (disablev6) &&
+           (AF_INET6 == pos->ai_family) )
         continue;
-      if ((IPPROTO_TCP != pos->ai_protocol) && (0 != pos->ai_protocol))
+      if ( (IPPROTO_TCP != pos->ai_protocol) &&
+           (0 != pos->ai_protocol) )
         continue;               /* not TCP */
-      if ((SOCK_STREAM != pos->ai_socktype) && (0 != pos->ai_socktype))
+      if ( (SOCK_STREAM != pos->ai_socktype) &&
+           (0 != pos->ai_socktype) )
         continue;               /* huh? */
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "Service `%s' will bind to `%s'\n",
-           service_name, GNUNET_a2s (pos->ai_addr, pos->ai_addrlen));
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Service `%s' will bind to `%s'\n",
+           service_name,
+           GNUNET_a2s (pos->ai_addr,
+                       pos->ai_addrlen));
       if (AF_INET == pos->ai_family)
       {
         GNUNET_assert (sizeof (struct sockaddr_in) == pos->ai_addrlen);
         saddrlens[i] = pos->ai_addrlen;
         saddrs[i] = GNUNET_malloc (saddrlens[i]);
-        GNUNET_memcpy (saddrs[i], pos->ai_addr, saddrlens[i]);
+        GNUNET_memcpy (saddrs[i],
+                       pos->ai_addr,
+                       saddrlens[i]);
         ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
       }
       else
@@ -765,7 +857,9 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
         GNUNET_assert (sizeof (struct sockaddr_in6) == pos->ai_addrlen);
         saddrlens[i] = pos->ai_addrlen;
         saddrs[i] = GNUNET_malloc (saddrlens[i]);
-        GNUNET_memcpy (saddrs[i], pos->ai_addr, saddrlens[i]);
+        GNUNET_memcpy (saddrs[i],
+                       pos->ai_addr,
+                       saddrlens[i]);
         ((struct sockaddr_in6 *) saddrs[i])->sin6_port = htons (port);
       }
       i++;
@@ -784,11 +878,16 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
       if (NULL != unixpath)
         resi++;
       i = 0;
-      saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
-      saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+      saddrs = GNUNET_new_array (resi + 1,
+                                 struct sockaddr *);
+      saddrlens = GNUNET_new_array (resi + 1,
+                                    socklen_t);
       if (NULL != unixpath)
       {
-        add_unixpath (saddrs, saddrlens, unixpath, abstract);
+        add_unixpath (saddrs,
+                      saddrlens,
+                      unixpath,
+                      abstract);
         i++;
       }
       saddrlens[i] = sizeof (struct sockaddr_in);
@@ -805,12 +904,17 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
       resi = 2;
       if (NULL != unixpath)
         resi++;
-      saddrs = GNUNET_malloc ((resi + 1) * sizeof (struct sockaddr *));
-      saddrlens = GNUNET_malloc ((resi + 1) * sizeof (socklen_t));
+      saddrs = GNUNET_new_array (resi + 1,
+                                 struct sockaddr *);
+      saddrlens = GNUNET_new_array (resi + 1,
+                                    socklen_t);
       i = 0;
       if (NULL != unixpath)
       {
-        add_unixpath (saddrs, saddrlens, unixpath, abstract);
+        add_unixpath (saddrs,
+                      saddrlens,
+                      unixpath,
+                      abstract);
         i++;
       }
       saddrlens[i] = sizeof (struct sockaddr_in6);
@@ -841,13 +945,14 @@ GNUNET_SERVICE_get_server_addresses (const char *service_name,
 /**
  * Read listen sockets from the parent process (ARM).
  *
- * @param sctx service context to initialize
- * @return #GNUNET_YES if ok, #GNUNET_NO if not ok (must bind yourself),
- * and #GNUNET_SYSERR on error.
+ * @param sh service context to initialize
+ * @return NULL-terminated array of sockets on success,
+ *         NULL if not ok (must bind yourself)
  */
-static int
-receive_sockets_from_parent (struct GNUNET_SERVICE_Context *sctx)
+static struct GNUNET_NETWORK_Handle **
+receive_sockets_from_parent (struct GNUNET_SERVICE_Handle *sh)
 {
+  static struct GNUNET_NETWORK_Handle **lsocks;
   const char *env_buf;
   int fail;
   uint64_t count;
@@ -855,15 +960,19 @@ receive_sockets_from_parent (struct GNUNET_SERVICE_Context *sctx)
   HANDLE lsocks_pipe;
 
   env_buf = getenv ("GNUNET_OS_READ_LSOCKS");
-  if ((NULL == env_buf) || (strlen (env_buf) <= 0))
-    return GNUNET_NO;
+  if ( (NULL == env_buf) ||
+       (strlen (env_buf) <= 0) )
+    return NULL;
   /* Using W32 API directly here, because this pipe will
    * never be used outside of this function, and it's just too much of a bother
    * to create a GNUnet API that boxes a HANDLE (the way it is done with socks)
    */
-  lsocks_pipe = (HANDLE) strtoul (env_buf, NULL, 10);
-  if ( (0 == lsocks_pipe) || (INVALID_HANDLE_VALUE == lsocks_pipe))
-    return GNUNET_NO;
+  lsocks_pipe = (HANDLE) strtoul (env_buf,
+                                  NULL,
+                                  10);
+  if ( (0 == lsocks_pipe) ||
+       (INVALID_HANDLE_VALUE == lsocks_pipe))
+    return NULL;
   fail = 1;
   do
   {
@@ -871,11 +980,17 @@ receive_sockets_from_parent (struct GNUNET_SERVICE_Context *sctx)
     int fail2;
     DWORD rd;
 
-    ret = ReadFile (lsocks_pipe, &count, sizeof (count), &rd, NULL);
-    if ((0 == ret) || (sizeof (count) != rd) || (0 == count))
+    ret = ReadFile (lsocks_pipe,
+                    &count,
+                    sizeof (count),
+                    &rd,
+                    NULL);
+    if ( (0 == ret) ||
+         (sizeof (count) != rd) ||
+         (0 == count) )
       break;
-    sctx->lsocks =
-        GNUNET_malloc (sizeof (struct GNUNET_NETWORK_Handle *) * (count + 1));
+    lsocks = GNUNET_new_array (count + 1,
+                               struct GNUNET_NETWORK_Handle *);
 
     fail2 = 1;
     for (i = 0; i < count; i++)
@@ -884,51 +999,165 @@ receive_sockets_from_parent (struct GNUNET_SERVICE_Context *sctx)
       uint64_t size;
       SOCKET s;
 
-      ret = ReadFile (lsocks_pipe, &size, sizeof (size), &rd, NULL);
-      if ( (0 == ret) || (sizeof (size) != rd) || (sizeof (pi) != size) )
+      ret = ReadFile (lsocks_pipe,
+                      &size,
+                      sizeof (size),
+                      &rd,
+                      NULL);
+      if ( (0 == ret) ||
+           (sizeof (size) != rd) ||
+           (sizeof (pi) != size) )
         break;
-      ret = ReadFile (lsocks_pipe, &pi, sizeof (pi), &rd, NULL);
-      if ( (0 == ret) || (sizeof (pi) != rd))
+      ret = ReadFile (lsocks_pipe,
+                      &pi,
+                      sizeof (pi),
+                      &rd,
+                      NULL);
+      if ( (0 == ret) ||
+           (sizeof (pi) != rd))
         break;
-      s = WSASocketA (pi.iAddressFamily, pi.iSocketType, pi.iProtocol, &pi, 0, WSA_FLAG_OVERLAPPED);
-      sctx->lsocks[i] = GNUNET_NETWORK_socket_box_native (s);
-      if (NULL == sctx->lsocks[i])
+      s = WSASocketA (pi.iAddressFamily,
+                      pi.iSocketType,
+                      pi.iProtocol,
+                      &pi,
+                      0,
+                      WSA_FLAG_OVERLAPPED);
+      lsocks[i] = GNUNET_NETWORK_socket_box_native (s);
+      if (NULL == lsocks[i])
         break;
       else if (i == count - 1)
         fail2 = 0;
     }
     if (fail2)
       break;
-    sctx->lsocks[count] = NULL;
+    lsocks[count] = NULL;
     fail = 0;
   }
   while (fail);
-
   CloseHandle (lsocks_pipe);
 
   if (fail)
   {
     LOG (GNUNET_ERROR_TYPE_ERROR,
          _("Could not access a pre-bound socket, will try to bind myself\n"));
-    for (i = 0; (i < count) && (NULL != sctx->lsocks[i]); i++)
-      GNUNET_break (0 == GNUNET_NETWORK_socket_close (sctx->lsocks[i]));
-    GNUNET_free_non_null (sctx->lsocks);
-    sctx->lsocks = NULL;
-    return GNUNET_NO;
+    for (i = 0; (i < count) && (NULL != lsocks[i]); i++)
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_NETWORK_socket_close (lsocks[i]));
+    GNUNET_free (lsocks);
+    return NULL;
   }
-  return GNUNET_YES;
+  return lsocks;
 }
 #endif
 
 
 /**
- * Setup addr, addrlen, idle_timeout
- * based on configuration!
+ * Create and initialize a listen socket for the server.
+ *
+ * @param server_addr address to listen on
+ * @param socklen length of @a server_addr
+ * @return NULL on error, otherwise the listen socket
+ */
+static struct GNUNET_NETWORK_Handle *
+open_listen_socket (const struct sockaddr *server_addr,
+                    socklen_t socklen)
+{
+  struct GNUNET_NETWORK_Handle *sock;
+  uint16_t port;
+  int eno;
+
+  switch (server_addr->sa_family)
+  {
+  case AF_INET:
+    port = ntohs (((const struct sockaddr_in *) server_addr)->sin_port);
+    break;
+  case AF_INET6:
+    port = ntohs (((const struct sockaddr_in6 *) server_addr)->sin6_port);
+    break;
+  case AF_UNIX:
+    port = 0;
+    break;
+  default:
+    GNUNET_break (0);
+    port = 0;
+    break;
+  }
+  sock = GNUNET_NETWORK_socket_create (server_addr->sa_family,
+                                       SOCK_STREAM,
+                                       0);
+  if (NULL == sock)
+  {
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "socket");
+    errno = 0;
+    return NULL;
+  }
+  /* bind the socket */
+  if (GNUNET_OK != GNUNET_NETWORK_socket_bind (sock,
+                                               server_addr,
+                                               socklen))
+  {
+    eno = errno;
+    if (EADDRINUSE != errno)
+    {
+      /* we don't log 'EADDRINUSE' here since an IPv4 bind may
+       * fail if we already took the port on IPv6; if both IPv4 and
+       * IPv6 binds fail, then our caller will log using the
+       * errno preserved in 'eno' */
+      LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                    "bind");
+      if (0 != port)
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             _("`%s' failed for port %d (%s).\n"),
+             "bind",
+             port,
+             (AF_INET == server_addr->sa_family) ? "IPv4" : "IPv6");
+      eno = 0;
+    }
+    else
+    {
+      if (0 != port)
+        LOG (GNUNET_ERROR_TYPE_WARNING,
+             _("`%s' failed for port %d (%s): address already in use\n"),
+             "bind", port,
+             (AF_INET == server_addr->sa_family) ? "IPv4" : "IPv6");
+      else if (AF_UNIX == server_addr->sa_family)
+      {
+        LOG (GNUNET_ERROR_TYPE_WARNING,
+             _("`%s' failed for `%s': address already in use\n"),
+             "bind",
+             GNUNET_a2s (server_addr, socklen));
+      }
+    }
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_NETWORK_socket_close (sock));
+    errno = eno;
+    return NULL;
+  }
+  if (GNUNET_OK != GNUNET_NETWORK_socket_listen (sock,
+                                                 5))
+  {
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "listen");
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_NETWORK_socket_close (sock));
+    errno = 0;
+    return NULL;
+  }
+  if (0 != port)
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Server starts to listen on port %u.\n",
+         port);
+  return sock;
+}
+
+
+/**
+ * Setup service handle
  *
  * Configuration may specify:
  * - PORT (where to bind to for TCP)
  * - UNIXPATH (where to bind to for UNIX domain sockets)
- * - TIMEOUT (after how many ms does an inactive service timeout);
  * - DISABLEV6 (disable support for IPv6, otherwise we use dual-stack)
  * - BINDTO (hostname or IP address to bind to, otherwise we take everything)
  * - ACCEPT_FROM  (only allow connections from specified IPv4 subnets)
@@ -936,108 +1165,170 @@ receive_sockets_from_parent (struct GNUNET_SERVICE_Context *sctx)
  * - REJECT_FROM  (disallow allow connections from specified IPv4 subnets)
  * - REJECT_FROM6 (disallow allow connections from specified IPv6 subnets)
  *
- * @param sctx service context to initialize
+ * @param sh service context to initialize
  * @return #GNUNET_OK if configuration succeeded
  */
 static int
-setup_service (struct GNUNET_SERVICE_Context *sctx)
+setup_service (struct GNUNET_SERVICE_Handle *sh)
 {
-  struct GNUNET_TIME_Relative idleout;
   int tolerant;
-
+  struct GNUNET_NETWORK_Handle **lsocks;
 #ifndef MINGW
   const char *nfds;
   unsigned int cnt;
   int flags;
 #endif
 
-  if (GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name, "TIMEOUT"))
-  {
-    if (GNUNET_OK !=
-        GNUNET_CONFIGURATION_get_value_time (sctx->cfg, sctx->service_name,
-                                             "TIMEOUT", &idleout))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Specified value for `%s' of service `%s' is invalid\n"),
-           "TIMEOUT", sctx->service_name);
-      return GNUNET_SYSERR;
-    }
-    sctx->timeout = idleout;
-  }
-  else
-    sctx->timeout = GNUNET_TIME_UNIT_FOREVER_REL;
-
   if (GNUNET_CONFIGURATION_have_value
-      (sctx->cfg, sctx->service_name, "TOLERANT"))
+      (sh->cfg,
+       sh->service_name,
+       "TOLERANT"))
   {
     if (GNUNET_SYSERR ==
         (tolerant =
-         GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+         GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
+                                               sh->service_name,
                                                "TOLERANT")))
     {
       LOG (GNUNET_ERROR_TYPE_ERROR,
            _("Specified value for `%s' of service `%s' is invalid\n"),
-           "TOLERANT", sctx->service_name);
+           "TOLERANT",
+           sh->service_name);
       return GNUNET_SYSERR;
     }
   }
   else
     tolerant = GNUNET_NO;
 
+  lsocks = NULL;
 #ifndef MINGW
   errno = 0;
-  if ((NULL != (nfds = getenv ("LISTEN_FDS"))) &&
-      (1 == SSCANF (nfds, "%u", &cnt)) && (cnt > 0) && (cnt < FD_SETSIZE) &&
-      (cnt + 4 < FD_SETSIZE))
+  if ( (NULL != (nfds = getenv ("LISTEN_FDS"))) &&
+       (1 == SSCANF (nfds,
+                     "%u",
+                     &cnt)) &&
+       (cnt > 0) &&
+       (cnt < FD_SETSIZE) &&
+       (cnt + 4 < FD_SETSIZE) )
   {
-    sctx->lsocks =
-        GNUNET_malloc (sizeof (struct GNUNET_NETWORK_Handle *) * (cnt + 1));
+    lsocks = GNUNET_new_array (cnt + 1,
+                               struct GNUNET_NETWORK_Handle *);
     while (0 < cnt--)
     {
-      flags = fcntl (3 + cnt, F_GETFD);
-      if ((flags < 0) || (0 != (flags & FD_CLOEXEC)) ||
-          (NULL ==
-           (sctx->lsocks[cnt] = GNUNET_NETWORK_socket_box_native (3 + cnt))))
+      flags = fcntl (3 + cnt,
+                     F_GETFD);
+      if ( (flags < 0) ||
+           (0 != (flags & FD_CLOEXEC)) ||
+           (NULL ==
+            (lsocks[cnt] = GNUNET_NETWORK_socket_box_native (3 + cnt))))
       {
         LOG (GNUNET_ERROR_TYPE_ERROR,
-             _
-             ("Could not access pre-bound socket %u, will try to bind myself\n"),
+             _("Could not access pre-bound socket %u, will try to bind myself\n"),
              (unsigned int) 3 + cnt);
         cnt++;
-        while (sctx->lsocks[cnt] != NULL)
-          GNUNET_break (0 == GNUNET_NETWORK_socket_close (sctx->lsocks[cnt++]));
-        GNUNET_free (sctx->lsocks);
-        sctx->lsocks = NULL;
+        while (NULL != lsocks[cnt])
+          GNUNET_break (GNUNET_OK ==
+                        GNUNET_NETWORK_socket_close (lsocks[cnt++]));
+        GNUNET_free (lsocks);
+        lsocks = NULL;
         break;
       }
     }
     unsetenv ("LISTEN_FDS");
   }
 #else
-  if (getenv ("GNUNET_OS_READ_LSOCKS") != NULL)
+  if (NULL != getenv ("GNUNET_OS_READ_LSOCKS"))
   {
-    receive_sockets_from_parent (sctx);
+    lsocks = receive_sockets_from_parent (sh);
     putenv ("GNUNET_OS_READ_LSOCKS=");
   }
 #endif
 
-  if ((NULL == sctx->lsocks) &&
-      (GNUNET_SYSERR ==
-       GNUNET_SERVICE_get_server_addresses (sctx->service_name, sctx->cfg,
-                                            &sctx->addrs, &sctx->addrlens)))
-    return GNUNET_SYSERR;
-  sctx->require_found = tolerant ? GNUNET_NO : GNUNET_YES;
-  sctx->match_uid =
-      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+  if (NULL != lsocks)
+  {
+    /* listen only on inherited sockets if we have any */
+    struct GNUNET_NETWORK_Handle **ls;
+
+    for (ls = lsocks; NULL != *ls; ls++)
+    {
+      struct ServiceListenContext *slc;
+
+      slc = GNUNET_new (struct ServiceListenContext);
+      slc->sh = sh;
+      slc->listen_socket = *ls;
+      GNUNET_CONTAINER_DLL_insert (sh->slc_head,
+                                   sh->slc_tail,
+                                   slc);
+    }
+    GNUNET_free (lsocks);
+  }
+  else
+  {
+    struct sockaddr **addrs;
+    socklen_t *addrlens;
+    int num;
+
+    num = get_server_addresses (sh->service_name,
+                                sh->cfg,
+                                &addrs,
+                                &addrlens);
+    if (GNUNET_SYSERR == num)
+      return GNUNET_SYSERR;
+
+    for (int i = 0; i < num; i++)
+    {
+      struct ServiceListenContext *slc;
+
+      slc = GNUNET_new (struct ServiceListenContext);
+      slc->sh = sh;
+      slc->listen_socket = open_listen_socket (addrs[i],
+                                               addrlens[i]);
+      if (NULL == slc->listen_socket)
+      {
+        GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
+                             "bind");
+        GNUNET_free (addrs[i++]);
+        GNUNET_free (slc);
+        continue;
+      }
+      GNUNET_free (addrs[i++]);
+      GNUNET_CONTAINER_DLL_insert (sh->slc_head,
+                                   sh->slc_tail,
+                                   slc);
+    }
+    GNUNET_free_non_null (addrlens);
+    GNUNET_free_non_null (addrs);
+    if ( (0 != num) &&
+         (NULL == sh->slc_head) )
+    {
+      /* All attempts to bind failed, hard failure */
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Could not bind to any of the ports I was supposed to, refusing to run!\n"));
+      return GNUNET_SYSERR;
+    }
+  }
+
+  sh->require_found = tolerant ? GNUNET_NO : GNUNET_YES;
+  sh->match_uid
+    = GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
+                                            sh->service_name,
                                             "UNIX_MATCH_UID");
-  sctx->match_gid =
-      GNUNET_CONFIGURATION_get_value_yesno (sctx->cfg, sctx->service_name,
+  sh->match_gid
+    = GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
+                                            sh->service_name,
                                             "UNIX_MATCH_GID");
-  process_acl4 (&sctx->v4_denied, sctx, "REJECT_FROM");
-  process_acl4 (&sctx->v4_allowed, sctx, "ACCEPT_FROM");
-  process_acl6 (&sctx->v6_denied, sctx, "REJECT_FROM6");
-  process_acl6 (&sctx->v6_allowed, sctx, "ACCEPT_FROM6");
-
+  process_acl4 (&sh->v4_denied,
+                sh,
+                "REJECT_FROM");
+  process_acl4 (&sh->v4_allowed,
+                sh,
+                "ACCEPT_FROM");
+  process_acl6 (&sh->v6_denied,
+                sh,
+                "REJECT_FROM6");
+  process_acl6 (&sh->v6_allowed,
+                sh,
+                "ACCEPT_FROM6");
   return GNUNET_OK;
 }
 
@@ -1046,185 +1337,129 @@ setup_service (struct GNUNET_SERVICE_Context *sctx)
  * Get the name of the user that'll be used
  * to provide the service.
  *
- * @param sctx service context
+ * @param sh service context
  * @return value of the 'USERNAME' option
  */
 static char *
-get_user_name (struct GNUNET_SERVICE_Context *sctx)
+get_user_name (struct GNUNET_SERVICE_Handle *sh)
 {
   char *un;
 
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (sctx->cfg, sctx->service_name,
-                                               "USERNAME", &un))
+      GNUNET_CONFIGURATION_get_value_filename (sh->cfg,
+                                               sh->service_name,
+                                               "USERNAME",
+                                               &un))
     return NULL;
   return un;
 }
 
 
 /**
- * Write PID file.
+ * Set user ID.
  *
- * @param sctx service context
- * @param pid PID to write (should be equal to 'getpid()'
- * @return  #GNUNET_OK on success (including no work to be done)
+ * @param sh service context
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
  */
 static int
-write_pid_file (struct GNUNET_SERVICE_Context *sctx, pid_t pid)
+set_user_id (struct GNUNET_SERVICE_Handle *sh)
 {
-  FILE *pidfd;
-  char *pif;
   char *user;
-  char *rdir;
-  int len;
-
-  if (NULL == (pif = get_pid_file_name (sctx)))
-    return GNUNET_OK;           /* no file desired */
-  user = get_user_name (sctx);
-  rdir = GNUNET_strdup (pif);
-  len = strlen (rdir);
-  while ((len > 0) && (rdir[len] != DIR_SEPARATOR))
-    len--;
-  rdir[len] = '\0';
-  if (0 != ACCESS (rdir, F_OK))
-  {
-    /* we get to create a directory -- and claim it
-     * as ours! */
-    (void) GNUNET_DISK_directory_create (rdir);
-    if ((NULL != user) && (0 < strlen (user)))
-      GNUNET_DISK_file_change_owner (rdir, user);
-  }
-  if (0 != ACCESS (rdir, W_OK | X_OK))
-  {
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_ERROR, "access", rdir);
-    GNUNET_free (rdir);
-    GNUNET_free_non_null (user);
-    GNUNET_free (pif);
+
+  if (NULL == (user = get_user_name (sh)))
+    return GNUNET_OK;           /* keep */
+#ifndef MINGW
+  struct passwd *pws;
+
+  errno = 0;
+  pws = getpwnam (user);
+  if (NULL == pws)
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         _("Cannot obtain information about user `%s': %s\n"),
+         user,
+         errno == 0 ? _("No such user") : STRERROR (errno));
+    GNUNET_free (user);
     return GNUNET_SYSERR;
   }
-  GNUNET_free (rdir);
-  pidfd = FOPEN (pif, "w");
-  if (NULL == pidfd)
+  if ( (0 != setgid (pws->pw_gid)) ||
+       (0 != setegid (pws->pw_gid)) ||
+#if HAVE_INITGROUPS
+       (0 != initgroups (user,
+                         pws->pw_gid)) ||
+#endif
+       (0 != setuid (pws->pw_uid)) ||
+       (0 != seteuid (pws->pw_uid)))
   {
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_ERROR, "fopen", pif);
-    GNUNET_free (pif);
-    GNUNET_free_non_null (user);
-    return GNUNET_SYSERR;
+    if ((0 != setregid (pws->pw_gid,
+                        pws->pw_gid)) ||
+        (0 != setreuid (pws->pw_uid,
+                        pws->pw_uid)))
+    {
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Cannot change user/group to `%s': %s\n"),
+           user,
+           STRERROR (errno));
+      GNUNET_free (user);
+      return GNUNET_SYSERR;
+    }
   }
-  if (0 > FPRINTF (pidfd, "%u", pid))
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "fprintf", pif);
-  GNUNET_break (0 == FCLOSE (pidfd));
-  if ((NULL != user) && (0 < strlen (user)))
-    GNUNET_DISK_file_change_owner (pif, user);
-  GNUNET_free_non_null (user);
-  GNUNET_free (pif);
+#endif
+  GNUNET_free (user);
   return GNUNET_OK;
 }
 
 
 /**
- * Task run during shutdown.  Stops the server/service.
+ * Get the name of the file where we will
+ * write the PID of the service.
  *
- * @param cls the `struct GNUNET_SERVICE_Context`
+ * @param sh service context
+ * @return name of the file for the process ID
  */
-static void
-shutdown_task (void *cls)
+static char *
+get_pid_file_name (struct GNUNET_SERVICE_Handle *sh)
 {
-  struct GNUNET_SERVICE_Context *service = cls;
-  struct GNUNET_SERVER_Handle *server = service->server;
+  char *pif;
 
-  service->shutdown_task = NULL;
-  if (0 != (service->options & GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN))
-    GNUNET_SERVER_stop_listening (server);
-  else
-    GNUNET_SERVER_destroy (server);
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (sh->cfg,
+                                               sh->service_name,
+                                               "PIDFILE",
+                                               &pif))
+    return NULL;
+  return pif;
 }
 
 
 /**
- * Initial task for the service.
+ * Delete the PID file that was created by our parent.
  *
- * @param cls service context
+ * @param sh service context
  */
 static void
-service_task (void *cls)
+pid_file_delete (struct GNUNET_SERVICE_Handle *sh)
 {
-  struct GNUNET_SERVICE_Context *sctx = cls;
-  unsigned int i;
+  char *pif = get_pid_file_name (sh);
 
-  (void) GNUNET_SPEEDUP_start_ (sctx->cfg);
-  GNUNET_RESOLVER_connect (sctx->cfg);
-  if (NULL != sctx->lsocks)
-    sctx->server
-      = GNUNET_SERVER_create_with_sockets (&check_access, sctx, sctx->lsocks,
-                                           sctx->timeout, sctx->require_found);
-  else
-    sctx->server
-      = GNUNET_SERVER_create (&check_access, sctx, sctx->addrs, sctx->addrlens,
-                              sctx->timeout, sctx->require_found);
-  if (NULL == sctx->server)
-  {
-    if (NULL != sctx->addrs)
-      for (i = 0; NULL != sctx->addrs[i]; i++)
-        LOG (GNUNET_ERROR_TYPE_INFO,
-             _("Failed to start `%s' at `%s'\n"),
-             sctx->service_name, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
-    sctx->ret = GNUNET_SYSERR;
-    return;
-  }
-#ifndef WINDOWS
-  if (NULL != sctx->addrs)
-    for (i = 0; NULL != sctx->addrs[i]; i++)
-      if ((AF_UNIX == sctx->addrs[i]->sa_family)
-          && ('\0' != ((const struct sockaddr_un *)sctx->addrs[i])->sun_path[0]))
-        GNUNET_DISK_fix_permissions (((const struct sockaddr_un *)sctx->addrs[i])->sun_path,
-                                     sctx->match_uid,
-                                     sctx->match_gid);
-#endif
-
-
-  if (0 == (sctx->options & GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN))
-  {
-    /* install a task that will kill the server
-     * process if the scheduler ever gets a shutdown signal */
-    sctx->shutdown_task = GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
-                                                         sctx);
-  }
-  sctx->my_handlers = GNUNET_malloc (sizeof (defhandlers));
-  GNUNET_memcpy (sctx->my_handlers, defhandlers, sizeof (defhandlers));
-  i = 0;
-  while (NULL != sctx->my_handlers[i].callback)
-    sctx->my_handlers[i++].callback_cls = sctx;
-  GNUNET_SERVER_add_handlers (sctx->server, sctx->my_handlers);
-  if (-1 != sctx->ready_confirm_fd)
-  {
-    GNUNET_break (1 == WRITE (sctx->ready_confirm_fd, ".", 1));
-    GNUNET_break (0 == CLOSE (sctx->ready_confirm_fd));
-    sctx->ready_confirm_fd = -1;
-    write_pid_file (sctx, getpid ());
-  }
-  if (NULL != sctx->addrs)
-  {
-    i = 0;
-    while (NULL != sctx->addrs[i])
-    {
-      LOG (GNUNET_ERROR_TYPE_INFO, _("Service `%s' runs at %s\n"),
-           sctx->service_name, GNUNET_a2s (sctx->addrs[i], sctx->addrlens[i]));
-      i++;
-    }
-  }
-  sctx->task (sctx->task_cls, sctx->server, sctx->cfg);
+  if (NULL == pif)
+    return;                     /* no PID file */
+  if (0 != UNLINK (pif))
+    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING,
+                       "unlink",
+                       pif);
+  GNUNET_free (pif);
 }
 
 
 /**
  * Detach from terminal.
  *
- * @param sctx service context
+ * @param sh service context
  * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
  */
 static int
-detach_terminal (struct GNUNET_SERVICE_Context *sctx)
+detach_terminal (struct GNUNET_SERVICE_Handle *sh)
 {
 #ifndef MINGW
   pid_t pid;
@@ -1233,13 +1468,15 @@ detach_terminal (struct GNUNET_SERVICE_Context *sctx)
 
   if (0 != PIPE (filedes))
   {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "pipe");
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "pipe");
     return GNUNET_SYSERR;
   }
   pid = fork ();
   if (pid < 0)
   {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "fork");
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "fork");
     return GNUNET_SYSERR;
   }
   if (0 != pid)
@@ -1249,15 +1486,19 @@ detach_terminal (struct GNUNET_SERVICE_Context *sctx)
 
     GNUNET_break (0 == CLOSE (filedes[1]));
     c = 'X';
-    if (1 != READ (filedes[0], &c, sizeof (char)))
-      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "read");
+    if (1 != READ (filedes[0],
+                   &c,
+                   sizeof (char)))
+      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING,
+                    "read");
     fflush (stdout);
     switch (c)
     {
     case '.':
       exit (0);
     case 'I':
-      LOG (GNUNET_ERROR_TYPE_INFO, _("Service process failed to initialize\n"));
+      LOG (GNUNET_ERROR_TYPE_INFO,
+           _("Service process failed to initialize\n"));
       break;
     case 'S':
       LOG (GNUNET_ERROR_TYPE_INFO,
@@ -1273,13 +1514,16 @@ detach_terminal (struct GNUNET_SERVICE_Context *sctx)
   GNUNET_break (0 == CLOSE (0));
   GNUNET_break (0 == CLOSE (1));
   GNUNET_break (0 == CLOSE (filedes[0]));
-  nullfd = OPEN ("/dev/null", O_RDWR | O_APPEND);
+  nullfd = OPEN ("/dev/null",
+                 O_RDWR | O_APPEND);
   if (nullfd < 0)
     return GNUNET_SYSERR;
   /* set stdin/stdout to /dev/null */
-  if ((dup2 (nullfd, 0) < 0) || (dup2 (nullfd, 1) < 0))
+  if ( (dup2 (nullfd, 0) < 0) ||
+       (dup2 (nullfd, 1) < 0) )
   {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "dup2");
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "dup2");
     (void) CLOSE (nullfd);
     return GNUNET_SYSERR;
   }
@@ -1287,8 +1531,9 @@ detach_terminal (struct GNUNET_SERVICE_Context *sctx)
   /* Detach from controlling terminal */
   pid = setsid ();
   if (-1 == pid)
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR, "setsid");
-  sctx->ready_confirm_fd = filedes[1];
+    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
+                  "setsid");
+  sh->ready_confirm_fd = filedes[1];
 #else
   /* FIXME: we probably need to do something else
    * elsewhere in order to fork the process itself... */
@@ -1299,144 +1544,228 @@ detach_terminal (struct GNUNET_SERVICE_Context *sctx)
 
 
 /**
- * Set user ID.
+ * Tear down the service, closing the listen sockets and
+ * freeing the ACLs.
  *
- * @param sctx service context
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
+ * @param sh handle to the service to tear down.
  */
-static int
-set_user_id (struct GNUNET_SERVICE_Context *sctx)
+static void
+teardown_service (struct GNUNET_SERVICE_Handle *sh)
 {
-  char *user;
-
-  if (NULL == (user = get_user_name (sctx)))
-    return GNUNET_OK;           /* keep */
-#ifndef MINGW
-  struct passwd *pws;
+  struct ServiceListenContext *slc;
 
-  errno = 0;
-  pws = getpwnam (user);
-  if (NULL == pws)
+  GNUNET_free_non_null (sh->v4_denied);
+  GNUNET_free_non_null (sh->v6_denied);
+  GNUNET_free_non_null (sh->v4_allowed);
+  GNUNET_free_non_null (sh->v6_allowed);
+  while (NULL != (slc = sh->slc_head))
   {
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         _("Cannot obtain information about user `%s': %s\n"), user,
-         errno == 0 ? _("No such user") : STRERROR (errno));
-    GNUNET_free (user);
-    return GNUNET_SYSERR;
+    GNUNET_CONTAINER_DLL_remove (sh->slc_head,
+                                 sh->slc_tail,
+                                 slc);
+    if (NULL != slc->listen_task)
+      GNUNET_SCHEDULER_cancel (slc->listen_task);
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_NETWORK_socket_close (slc->listen_socket));
+    GNUNET_free (slc);
   }
-  if ((0 != setgid (pws->pw_gid)) || (0 != setegid (pws->pw_gid)) ||
-#if HAVE_INITGROUPS
-      (0 != initgroups (user, pws->pw_gid)) ||
-#endif
-      (0 != setuid (pws->pw_uid)) || (0 != seteuid (pws->pw_uid)))
+}
+
+
+/**
+ * Low-level function to start a service if the scheduler
+ * is already running.  Should only be used directly in
+ * special cases.
+ *
+ * The function will launch the service with the name @a service_name
+ * using the @a service_options to configure its shutdown
+ * behavior. When clients connect or disconnect, the respective
+ * @a connect_cb or @a disconnect_cb functions will be called. For
+ * messages received from the clients, the respective @a handlers will
+ * be invoked; for the closure of the handlers we use the return value
+ * from the @a connect_cb invocation of the respective client.
+ *
+ * Each handler MUST call #GNUNET_SERVICE_client_continue() after each
+ * message to receive further messages from this client.  If
+ * #GNUNET_SERVICE_client_continue() is not called within a short
+ * time, a warning will be logged. If delays are expected, services
+ * should call #GNUNET_SERVICE_client_disable_continue_warning() to
+ * disable the warning.
+ *
+ * Clients sending invalid messages (based on @a handlers) will be
+ * dropped. Additionally, clients can be dropped at any time using
+ * #GNUNET_SERVICE_client_drop().
+ *
+ * The service must be stopped using #GNUNET_SERVICE_stop().
+ *
+ * @param service_name name of the service to run
+ * @param cfg configuration to use
+ * @param connect_cb function to call whenever a client connects
+ * @param disconnect_cb function to call whenever a client disconnects
+ * @param cls closure argument for @a connect_cb and @a disconnect_cb
+ * @param handlers NULL-terminated array of message handlers for the service,
+ *                 the closure will be set to the value returned by
+ *                 the @a connect_cb for the respective connection
+ * @return NULL on error
+ */
+struct GNUNET_SERVICE_Handle *
+GNUNET_SERVICE_start (const char *service_name,
+                      const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      GNUNET_SERVICE_ConnectHandler connect_cb,
+                      GNUNET_SERVICE_DisconnectHandler disconnect_cb,
+                      void *cls,
+                      const struct GNUNET_MQ_MessageHandler *handlers)
+{
+  struct GNUNET_SERVICE_Handle *sh;
+
+  sh = GNUNET_new (struct GNUNET_SERVICE_Handle);
+  sh->service_name = service_name;
+  sh->cfg = cfg;
+  sh->connect_cb = connect_cb;
+  sh->disconnect_cb = disconnect_cb;
+  sh->cb_cls = cls;
+  sh->handlers = GNUNET_MQ_copy_handlers (handlers);
+  if (GNUNET_OK != setup_service (sh))
   {
-    if ((0 != setregid (pws->pw_gid, pws->pw_gid)) ||
-        (0 != setreuid (pws->pw_uid, pws->pw_uid)))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR, _("Cannot change user/group to `%s': %s\n"),
-           user, STRERROR (errno));
-      GNUNET_free (user);
-      return GNUNET_SYSERR;
-    }
+    GNUNET_free_non_null (sh->handlers);
+    GNUNET_free (sh);
+    return NULL;
   }
-#endif
-  GNUNET_free (user);
-  return GNUNET_OK;
+  GNUNET_SERVICE_resume (sh);
+  return sh;
 }
 
 
 /**
- * Delete the PID file that was created by our parent.
+ * Stops a service that was started with #GNUNET_SERVICE_start().
  *
- * @param sctx service context
+ * @param srv service to stop
  */
-static void
-pid_file_delete (struct GNUNET_SERVICE_Context *sctx)
+void
+GNUNET_SERVICE_stop (struct GNUNET_SERVICE_Handle *srv)
 {
-  char *pif = get_pid_file_name (sctx);
-
-  if (NULL == pif)
-    return;                     /* no PID file */
-  if (0 != UNLINK (pif))
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING, "unlink", pif);
-  GNUNET_free (pif);
+  struct GNUNET_SERVICE_Client *client;
+
+  GNUNET_SERVICE_suspend (srv);
+  while (NULL != (client = srv->clients_head))
+    GNUNET_SERVICE_client_drop (client);
+  teardown_service (srv);
+  GNUNET_free_non_null (srv->handlers);
+  GNUNET_free (srv);
 }
 
 
 /**
- * Run a standard GNUnet service startup sequence (initialize loggers
- * and configuration, parse options).
+ * Creates the "main" function for a GNUnet service.  You
+ * should almost always use the #GNUNET_SERVICE_MAIN macro
+ * instead of calling this function directly (except
+ * for ARM, which should call this function directly).
+ *
+ * The function will launch the service with the name @a service_name
+ * using the @a service_options to configure its shutdown
+ * behavior. Once the service is ready, the @a init_cb will be called
+ * for service-specific initialization.  @a init_cb will be given the
+ * service handler which can be used to control the service's
+ * availability.  When clients connect or disconnect, the respective
+ * @a connect_cb or @a disconnect_cb functions will be called. For
+ * messages received from the clients, the respective @a handlers will
+ * be invoked; for the closure of the handlers we use the return value
+ * from the @a connect_cb invocation of the respective client.
  *
- * @param argc number of command line arguments
- * @param argv command line arguments
- * @param service_name our service name
- * @param options service options
- * @param task main task of the service
- * @param task_cls closure for @a task
- * @return #GNUNET_SYSERR on error, #GNUNET_OK
- *         if we shutdown nicely
+ * Each handler MUST call #GNUNET_SERVICE_client_continue() after each
+ * message to receive further messages from this client.  If
+ * #GNUNET_SERVICE_client_continue() is not called within a short
+ * time, a warning will be logged. If delays are expected, services
+ * should call #GNUNET_SERVICE_client_disable_continue_warning() to
+ * disable the warning.
+ *
+ * Clients sending invalid messages (based on @a handlers) will be
+ * dropped. Additionally, clients can be dropped at any time using
+ * #GNUNET_SERVICE_client_drop().
+ *
+ * @param argc number of command-line arguments in @a argv
+ * @param argv array of command-line arguments
+ * @param service_name name of the service to run
+ * @param options options controlling shutdown of the service
+ * @param service_init_cb function to call once the service is ready
+ * @param connect_cb function to call whenever a client connects
+ * @param disconnect_cb function to call whenever a client disconnects
+ * @param cls closure argument for @a service_init_cb, @a connect_cb and @a disconnect_cb
+ * @param handlers NULL-terminated array of message handlers for the service,
+ *                 the closure will be set to the value returned by
+ *                 the @a connect_cb for the respective connection
+ * @return 0 on success, non-zero on error
  */
 int
-GNUNET_SERVICE_run (int argc, char *const *argv,
-                    const char *service_name,
-                    enum GNUNET_SERVICE_Options options,
-                    GNUNET_SERVICE_Main task,
-                    void *task_cls)
+GNUNET_SERVICE_run_ (int argc,
+                     char *const *argv,
+                     const char *service_name,
+                     enum GNUNET_SERVICE_Options options,
+                     GNUNET_SERVICE_InitCallback service_init_cb,
+                     GNUNET_SERVICE_ConnectHandler connect_cb,
+                     GNUNET_SERVICE_DisconnectHandler disconnect_cb,
+                     void *cls,
+                     const struct GNUNET_MQ_MessageHandler *handlers)
 {
-#define HANDLE_ERROR do { GNUNET_break (0); goto shutdown; } while (0)
-
-  int err;
-  int ret;
-  char *cfg_fn;
-  char *opt_cfg_fn;
+  struct GNUNET_SERVICE_Handle sh;
+  char *cfg_filename;
+  char *opt_cfg_filename;
   char *loglev;
+  const char *xdg;
   char *logfile;
   int do_daemonize;
-  unsigned int i;
   unsigned long long skew_offset;
   unsigned long long skew_variance;
   long long clock_offset;
-  struct GNUNET_SERVICE_Context sctx;
   struct GNUNET_CONFIGURATION_Handle *cfg;
-  const char *xdg;
+  int ret;
+  int err;
 
   struct GNUNET_GETOPT_CommandLineOption service_options[] = {
-    GNUNET_GETOPT_OPTION_CFG_FILE (&opt_cfg_fn),
-    {'d', "daemonize", NULL,
-     gettext_noop ("do daemonize (detach from terminal)"), 0,
-     GNUNET_GETOPT_set_one, &do_daemonize},
+    GNUNET_GETOPT_OPTION_CFG_FILE (&opt_cfg_filename),
+    GNUNET_GETOPT_OPTION_SET_ONE ('d',
+                                  "daemonize",
+                                  gettext_noop ("do daemonize (detach from terminal)"),
+                                  &do_daemonize),
     GNUNET_GETOPT_OPTION_HELP (NULL),
     GNUNET_GETOPT_OPTION_LOGLEVEL (&loglev),
     GNUNET_GETOPT_OPTION_LOGFILE (&logfile),
     GNUNET_GETOPT_OPTION_VERSION (PACKAGE_VERSION " " VCS_VERSION),
     GNUNET_GETOPT_OPTION_END
   };
+
   err = 1;
-  do_daemonize = 0;
-  logfile = NULL;
-  loglev = NULL;
-  opt_cfg_fn = NULL;
+  memset (&sh,
+          0,
+          sizeof (sh));
   xdg = getenv ("XDG_CONFIG_HOME");
   if (NULL != xdg)
-    GNUNET_asprintf (&cfg_fn,
+    GNUNET_asprintf (&cfg_filename,
                      "%s%s%s",
                      xdg,
                      DIR_SEPARATOR_STR,
                      GNUNET_OS_project_data_get ()->config_file);
   else
-    cfg_fn = GNUNET_strdup (GNUNET_OS_project_data_get ()->user_config_file);
-  memset (&sctx, 0, sizeof (sctx));
-  sctx.options = options;
-  sctx.ready_confirm_fd = -1;
-  sctx.ret = GNUNET_OK;
-  sctx.timeout = GNUNET_TIME_UNIT_FOREVER_REL;
-  sctx.task = task;
-  sctx.task_cls = task_cls;
-  sctx.service_name = service_name;
-  sctx.cfg = cfg = GNUNET_CONFIGURATION_create ();
+    cfg_filename = GNUNET_strdup (GNUNET_OS_project_data_get ()->user_config_file);
+  sh.ready_confirm_fd = -1;
+  sh.options = options;
+  sh.cfg = cfg = GNUNET_CONFIGURATION_create ();
+  sh.service_init_cb = service_init_cb;
+  sh.connect_cb = connect_cb;
+  sh.disconnect_cb = disconnect_cb;
+  sh.cb_cls = cls;
+  sh.handlers = GNUNET_MQ_copy_handlers (handlers);
+  sh.service_name = service_name;
 
   /* setup subsystems */
-  ret = GNUNET_GETOPT_run (service_name, service_options, argc, argv);
+  loglev = NULL;
+  logfile = NULL;
+  opt_cfg_filename = NULL;
+  do_daemonize = 0;
+  ret = GNUNET_GETOPT_run (service_name,
+                           service_options,
+                           argc,
+                           argv);
   if (GNUNET_SYSERR == ret)
     goto shutdown;
   if (GNUNET_NO == ret)
@@ -1444,254 +1773,844 @@ GNUNET_SERVICE_run (int argc, char *const *argv,
     err = 0;
     goto shutdown;
   }
-  if (GNUNET_OK != GNUNET_log_setup (service_name, loglev, logfile))
-    HANDLE_ERROR;
-  if (NULL == opt_cfg_fn)
-    opt_cfg_fn = GNUNET_strdup (cfg_fn);
-  if (GNUNET_YES == GNUNET_DISK_file_test (opt_cfg_fn))
+  if (GNUNET_OK != GNUNET_log_setup (service_name,
+                                     loglev,
+                                     logfile))
+  {
+    GNUNET_break (0);
+    goto shutdown;
+  }
+  if (NULL == opt_cfg_filename)
+    opt_cfg_filename = GNUNET_strdup (cfg_filename);
+  if (GNUNET_YES == GNUNET_DISK_file_test (opt_cfg_filename))
   {
-    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg, opt_cfg_fn))
+    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg,
+                                                    opt_cfg_filename))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   _("Malformed configuration file `%s', exit ...\n"),
-                  opt_cfg_fn);
+                  opt_cfg_filename);
       goto shutdown;
     }
   }
   else
   {
-    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg, NULL))
+    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg,
+                                                    NULL))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   _("Malformed configuration, exit ...\n"));
       goto shutdown;
     }
-    if (0 != strcmp (opt_cfg_fn, cfg_fn))
+    if (0 != strcmp (opt_cfg_filename,
+                     cfg_filename))
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   _("Could not access configuration file `%s'\n"),
-                  opt_cfg_fn);
+                  opt_cfg_filename);
   }
-  if (GNUNET_OK != setup_service (&sctx))
+  if (GNUNET_OK != setup_service (&sh))
     goto shutdown;
-  if ((1 == do_daemonize) && (GNUNET_OK != detach_terminal (&sctx)))
-    HANDLE_ERROR;
-  if (GNUNET_OK != set_user_id (&sctx))
+  if ( (1 == do_daemonize) &&
+       (GNUNET_OK != detach_terminal (&sh)) )
+  {
+    GNUNET_break (0);
+    goto shutdown;
+  }
+  if (GNUNET_OK != set_user_id (&sh))
     goto shutdown;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Service `%s' runs with configuration from `%s'\n",
        service_name,
-       opt_cfg_fn);
+       opt_cfg_filename);
   if ((GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_number (sctx.cfg, "TESTING",
-                                              "SKEW_OFFSET", &skew_offset)) &&
+       GNUNET_CONFIGURATION_get_value_number (sh.cfg,
+                                              "TESTING",
+                                              "SKEW_OFFSET",
+                                              &skew_offset)) &&
       (GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_number (sctx.cfg, "TESTING",
-                                              "SKEW_VARIANCE", &skew_variance)))
+       GNUNET_CONFIGURATION_get_value_number (sh.cfg,
+                                              "TESTING",
+                                              "SKEW_VARIANCE",
+                                              &skew_variance)))
   {
     clock_offset = skew_offset - skew_variance;
     GNUNET_TIME_set_offset (clock_offset);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Skewing clock by %dll ms\n", clock_offset);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Skewing clock by %dll ms\n",
+         clock_offset);
   }
+  GNUNET_RESOLVER_connect (sh.cfg);
+
   /* actually run service */
   err = 0;
-  GNUNET_SCHEDULER_run (&service_task, &sctx);
+  GNUNET_SCHEDULER_run (&service_main,
+                        &sh);
   /* shutdown */
-  if ((1 == do_daemonize) && (NULL != sctx.server))
-    pid_file_delete (&sctx);
-  GNUNET_free_non_null (sctx.my_handlers);
+  if (1 == do_daemonize)
+    pid_file_delete (&sh);
 
 shutdown:
-  if (-1 != sctx.ready_confirm_fd)
+  if (-1 != sh.ready_confirm_fd)
   {
-    if (1 != WRITE (sctx.ready_confirm_fd, err ? "I" : "S", 1))
-      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING, "write");
-    GNUNET_break (0 == CLOSE (sctx.ready_confirm_fd));
+    if (1 != WRITE (sh.ready_confirm_fd,
+                    err ? "I" : "S",
+                    1))
+      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING,
+                    "write");
+    GNUNET_break (0 == CLOSE (sh.ready_confirm_fd));
   }
 #if HAVE_MALLINFO
   {
     char *counter;
 
     if ( (GNUNET_YES ==
-          GNUNET_CONFIGURATION_have_value (sctx.cfg, service_name,
+          GNUNET_CONFIGURATION_have_value (sh.cfg,
+                                           service_name,
                                            "GAUGER_HEAP")) &&
          (GNUNET_OK ==
-          GNUNET_CONFIGURATION_get_value_string (sctx.cfg, service_name,
+          GNUNET_CONFIGURATION_get_value_string (sh.cfg,
+                                                 service_name,
                                                  "GAUGER_HEAP",
                                                  &counter)) )
     {
       struct mallinfo mi;
 
       mi = mallinfo ();
-      GAUGER (service_name, counter, mi.usmblks, "blocks");
+      GAUGER (service_name,
+              counter,
+              mi.usmblks,
+              "blocks");
       GNUNET_free (counter);
     }
   }
 #endif
+  teardown_service (&sh);
+  GNUNET_free_non_null (sh.handlers);
   GNUNET_SPEEDUP_stop_ ();
   GNUNET_CONFIGURATION_destroy (cfg);
-  i = 0;
-  if (NULL != sctx.addrs)
-    while (NULL != sctx.addrs[i])
-      GNUNET_free (sctx.addrs[i++]);
-  GNUNET_free_non_null (sctx.addrs);
-  GNUNET_free_non_null (sctx.addrlens);
   GNUNET_free_non_null (logfile);
   GNUNET_free_non_null (loglev);
-  GNUNET_free (cfg_fn);
-  GNUNET_free_non_null (opt_cfg_fn);
-  GNUNET_free_non_null (sctx.v4_denied);
-  GNUNET_free_non_null (sctx.v6_denied);
-  GNUNET_free_non_null (sctx.v4_allowed);
-  GNUNET_free_non_null (sctx.v6_allowed);
-
-  return err ? GNUNET_SYSERR : sctx.ret;
+  GNUNET_free (cfg_filename);
+  GNUNET_free_non_null (opt_cfg_filename);
+
+  return err ? GNUNET_SYSERR : sh.ret;
 }
 
 
 /**
- * Run a service startup sequence within an existing
- * initialized system.
+ * Suspend accepting connections from the listen socket temporarily.
+ * Resume activity using #GNUNET_SERVICE_resume.
  *
- * @param service_name our service name
- * @param cfg configuration to use
- * @param options service options
- * @return NULL on error, service handle
+ * @param sh service to stop accepting connections.
  */
-struct GNUNET_SERVICE_Context *
-GNUNET_SERVICE_start (const char *service_name,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      enum GNUNET_SERVICE_Options options)
+void
+GNUNET_SERVICE_suspend (struct GNUNET_SERVICE_Handle *sh)
 {
-  int i;
-  struct GNUNET_SERVICE_Context *sctx;
+  struct ServiceListenContext *slc;
 
-  sctx = GNUNET_new (struct GNUNET_SERVICE_Context);
-  sctx->ready_confirm_fd = -1;  /* no daemonizing */
-  sctx->ret = GNUNET_OK;
-  sctx->timeout = GNUNET_TIME_UNIT_FOREVER_REL;
-  sctx->service_name = service_name;
-  sctx->cfg = cfg;
-  sctx->options = options;
+  for (slc = sh->slc_head; NULL != slc; slc = slc->next)
+  {
+    if (NULL != slc->listen_task)
+    {
+      GNUNET_SCHEDULER_cancel (slc->listen_task);
+      slc->listen_task = NULL;
+    }
+  }
+}
 
-  /* setup subsystems */
-  if (GNUNET_OK != setup_service (sctx))
+
+/**
+ * Task run when we are ready to transmit data to the
+ * client.
+ *
+ * @param cls the `struct GNUNET_SERVICE_Client *` to send to
+ */
+static void
+do_send (void *cls)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+  ssize_t ret;
+  size_t left;
+  const char *buf;
+
+  client->send_task = NULL;
+  buf = (const char *) client->msg;
+  left = ntohs (client->msg->size) - client->msg_pos;
+  ret = GNUNET_NETWORK_socket_send (client->sock,
+                                    &buf[client->msg_pos],
+                                    left);
+  GNUNET_assert (ret <= (ssize_t) left);
+  if (0 == ret)
   {
-    GNUNET_SERVICE_stop (sctx);
-    return NULL;
+    GNUNET_MQ_inject_error (client->mq,
+                            GNUNET_MQ_ERROR_WRITE);
+    return;
   }
-  if (NULL != sctx->lsocks)
-    sctx->server =
-        GNUNET_SERVER_create_with_sockets (&check_access, sctx, sctx->lsocks,
-                                           sctx->timeout, sctx->require_found);
-  else
-    sctx->server =
-        GNUNET_SERVER_create (&check_access, sctx, sctx->addrs, sctx->addrlens,
-                              sctx->timeout, sctx->require_found);
+  if (-1 == ret)
+  {
+    if ( (EAGAIN == errno) ||
+         (EINTR == errno) )
+    {
+      /* ignore */
+      ret = 0;
+    }
+    else
+    {
+      if (EPIPE != errno)
+        GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
+                             "send");
+      GNUNET_MQ_inject_error (client->mq,
+                              GNUNET_MQ_ERROR_WRITE);
+      return;
+    }
+  }
+  if (0 == client->msg_pos)
+  {
+    GNUNET_MQ_impl_send_in_flight (client->mq);
+  }
+  client->msg_pos += ret;
+  if (left > ret)
+  {
+    GNUNET_assert (NULL == client->drop_task);
+    client->send_task
+      = GNUNET_SCHEDULER_add_write_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                        client->sock,
+                                        &do_send,
+                                        client);
+    return;
+  }
+  GNUNET_MQ_impl_send_continue (client->mq);
+}
+
 
-  if (NULL == sctx->server)
+/**
+ * Signature of functions implementing the sending functionality of a
+ * message queue.
+ *
+ * @param mq the message queue
+ * @param msg the message to send
+ * @param impl_state our `struct GNUNET_SERVICE_Client *`
+ */
+static void
+service_mq_send (struct GNUNET_MQ_Handle *mq,
+                 const struct GNUNET_MessageHeader *msg,
+                 void *impl_state)
+{
+  struct GNUNET_SERVICE_Client *client = impl_state;
+
+  if (NULL != client->drop_task)
+    return; /* we're going down right now, do not try to send */
+  GNUNET_assert (NULL == client->send_task);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending message of type %u and size %u to client\n",
+       ntohs (msg->type),
+       ntohs (msg->size));
+  client->msg = msg;
+  client->msg_pos = 0;
+  client->send_task
+    = GNUNET_SCHEDULER_add_write_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                      client->sock,
+                                      &do_send,
+                                      client);
+}
+
+
+/**
+ * Implementation function that cancels the currently sent message.
+ *
+ * @param mq message queue
+ * @param impl_state state specific to the implementation
+ */
+static void
+service_mq_cancel (struct GNUNET_MQ_Handle *mq,
+                   void *impl_state)
+{
+  struct GNUNET_SERVICE_Client *client = impl_state;
+
+  GNUNET_assert (0 == client->msg_pos);
+  client->msg = NULL;
+  GNUNET_SCHEDULER_cancel (client->send_task);
+  client->send_task = NULL;
+}
+
+
+/**
+ * Generic error handler, called with the appropriate
+ * error code and the same closure specified at the creation of
+ * the message queue.
+ * Not every message queue implementation supports an error handler.
+ *
+ * @param cls closure with our `struct GNUNET_SERVICE_Client`
+ * @param error error code
+ */
+static void
+service_mq_error_handler (void *cls,
+                          enum GNUNET_MQ_Error error)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+  struct GNUNET_SERVICE_Handle *sh = client->sh;
+
+  if ( (GNUNET_MQ_ERROR_NO_MATCH == error) &&
+       (GNUNET_NO == sh->require_found) )
   {
-    GNUNET_SERVICE_stop (sctx);
-    return NULL;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "No handler for message of type %u found\n",
+                (unsigned int) client->warn_type);
+    GNUNET_SERVICE_client_continue (client);
+    return; /* ignore error */
+  }
+  GNUNET_SERVICE_client_drop (client);
+}
+
+
+/**
+ * Task run to warn about missing calls to #GNUNET_SERVICE_client_continue().
+ *
+ * @param cls our `struct GNUNET_SERVICE_Client *` to process more requests from
+ */
+static void
+warn_no_client_continue (void *cls)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+
+  GNUNET_break (0 != client->warn_type); /* type should never be 0 here, as we don't use 0 */
+  client->warn_task
+    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+                                    &warn_no_client_continue,
+                                    client);
+  LOG (GNUNET_ERROR_TYPE_WARNING,
+       _("Processing code for message of type %u did not call `GNUNET_SERVICE_client_continue' after %s\n"),
+       (unsigned int) client->warn_type,
+       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (client->warn_start),
+                                               GNUNET_YES));
+}
+
+
+/**
+ * Functions with this signature are called whenever a
+ * complete message is received by the tokenizer for a client.
+ *
+ * Do not call #GNUNET_MST_destroy() from within
+ * the scope of this callback.
+ *
+ * @param cls closure with the `struct GNUNET_SERVICE_Client *`
+ * @param message the actual message
+ * @return #GNUNET_OK on success, #GNUNET_SYSERR if the client was dropped
+ */
+static int
+service_client_mst_cb (void *cls,
+                       const struct GNUNET_MessageHeader *message)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received message of type %u and size %u from client\n",
+       ntohs (message->type),
+       ntohs (message->size));
+  GNUNET_assert (GNUNET_NO == client->needs_continue);
+  client->needs_continue = GNUNET_YES;
+  client->warn_type = ntohs (message->type);
+  client->warn_start = GNUNET_TIME_absolute_get ();
+  GNUNET_assert (NULL == client->warn_task);
+  client->warn_task
+    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
+                                    &warn_no_client_continue,
+                                    client);
+  GNUNET_MQ_inject_message (client->mq,
+                            message);
+  if (NULL != client->drop_task)
+    return GNUNET_SYSERR;
+  return GNUNET_OK;
+}
+
+
+/**
+ * A client sent us data. Receive and process it.  If we are done,
+ * reschedule this task.
+ *
+ * @param cls the `struct GNUNET_SERVICE_Client` that sent us data.
+ */
+static void
+service_client_recv (void *cls)
+{
+  struct GNUNET_SERVICE_Client *client = cls;
+  int ret;
+
+  client->recv_task = NULL;
+  ret = GNUNET_MST_read (client->mst,
+                         client->sock,
+                         GNUNET_NO,
+                         GNUNET_YES);
+  if (GNUNET_SYSERR == ret)
+  {
+    /* client closed connection (or IO error) */
+    if (NULL == client->drop_task)
+    {
+      GNUNET_assert (GNUNET_NO == client->needs_continue);
+      GNUNET_SERVICE_client_drop (client);
+    }
+    return;
+  }
+  if (GNUNET_NO == ret)
+    return; /* more messages in buffer, wait for application
+               to be done processing */
+  GNUNET_assert (GNUNET_OK == ret);
+  if (GNUNET_YES == client->needs_continue)
+    return;
+  if (NULL != client->recv_task)
+    return;
+  /* MST needs more data, re-schedule read job */
+  client->recv_task
+    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                     client->sock,
+                                     &service_client_recv,
+                                     client);
+}
+
+
+/**
+ * We have successfully accepted a connection from a client.  Now
+ * setup the client (with the scheduler) and tell the application.
+ *
+ * @param sh service that accepted the client
+ * @param sock socket associated with the client
+ */
+static void
+start_client (struct GNUNET_SERVICE_Handle *sh,
+              struct GNUNET_NETWORK_Handle *csock)
+{
+  struct GNUNET_SERVICE_Client *client;
+
+  client = GNUNET_new (struct GNUNET_SERVICE_Client);
+  GNUNET_CONTAINER_DLL_insert (sh->clients_head,
+                               sh->clients_tail,
+                               client);
+  client->sh = sh;
+  client->sock = csock;
+  client->mq = GNUNET_MQ_queue_for_callbacks (&service_mq_send,
+                                              NULL,
+                                              &service_mq_cancel,
+                                              client,
+                                              sh->handlers,
+                                              &service_mq_error_handler,
+                                              client);
+  client->mst = GNUNET_MST_create (&service_client_mst_cb,
+                                   client);
+  if (NULL != sh->connect_cb)
+    client->user_context = sh->connect_cb (sh->cb_cls,
+                                           client,
+                                           client->mq);
+  GNUNET_MQ_set_handlers_closure (client->mq,
+                                  client->user_context);
+  client->recv_task
+    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                     client->sock,
+                                     &service_client_recv,
+                                     client);
+}
+
+
+/**
+ * Check if the given IP address is in the list of IP addresses.
+ *
+ * @param list a list of networks
+ * @param add the IP to check (in network byte order)
+ * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
+ */
+static int
+check_ipv4_listed (const struct GNUNET_STRINGS_IPv4NetworkPolicy *list,
+                   const struct in_addr *add)
+{
+  unsigned int i;
+
+  if (NULL == list)
+    return GNUNET_NO;
+  i = 0;
+  while ( (0 != list[i].network.s_addr) ||
+          (0 != list[i].netmask.s_addr) )
+  {
+    if ((add->s_addr & list[i].netmask.s_addr) ==
+        (list[i].network.s_addr & list[i].netmask.s_addr))
+      return GNUNET_YES;
+    i++;
+  }
+  return GNUNET_NO;
+}
+
+
+/**
+ * Check if the given IP address is in the list of IP addresses.
+ *
+ * @param list a list of networks
+ * @param ip the IP to check (in network byte order)
+ * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
+ */
+static int
+check_ipv6_listed (const struct GNUNET_STRINGS_IPv6NetworkPolicy *list,
+                   const struct in6_addr *ip)
+{
+  unsigned int i;
+  unsigned int j;
+  struct in6_addr zero;
+
+  if (NULL == list)
+    return GNUNET_NO;
+  memset (&zero,
+          0,
+          sizeof (struct in6_addr));
+  i = 0;
+NEXT:
+  while (0 != memcmp (&zero,
+                      &list[i].network,
+                      sizeof (struct in6_addr)))
+  {
+    for (j = 0; j < sizeof (struct in6_addr) / sizeof (int); j++)
+      if (((((int *) ip)[j] & ((int *) &list[i].netmask)[j])) !=
+          (((int *) &list[i].network)[j] & ((int *) &list[i].netmask)[j]))
+      {
+        i++;
+        goto NEXT;
+      }
+    return GNUNET_YES;
   }
+  return GNUNET_NO;
+}
+
+
+/**
+ * We have a client. Accept the incoming socket(s) (and reschedule
+ * the listen task).
+ *
+ * @param cls the `struct ServiceListenContext` of the ready listen socket
+ */
+static void
+accept_client (void *cls)
+{
+  struct ServiceListenContext *slc = cls;
+  struct GNUNET_SERVICE_Handle *sh = slc->sh;
+
+  slc->listen_task = NULL;
+  while (1)
+  {
+    struct GNUNET_NETWORK_Handle *sock;
+    const struct sockaddr_in *v4;
+    const struct sockaddr_in6 *v6;
+    struct sockaddr_storage sa;
+    socklen_t addrlen;
+    int ok;
+
+    addrlen = sizeof (sa);
+    sock = GNUNET_NETWORK_socket_accept (slc->listen_socket,
+                                         (struct sockaddr *) &sa,
+                                         &addrlen);
+    if (NULL == sock)
+      break;
+    switch (sa.ss_family)
+    {
+    case AF_INET:
+      GNUNET_assert (addrlen == sizeof (struct sockaddr_in));
+      v4 = (const struct sockaddr_in *) &sa;
+      ok = ( ( (NULL == sh->v4_allowed) ||
+               (check_ipv4_listed (sh->v4_allowed,
+                                   &v4->sin_addr))) &&
+             ( (NULL == sh->v4_denied) ||
+               (! check_ipv4_listed (sh->v4_denied,
+                                     &v4->sin_addr)) ) );
+      break;
+    case AF_INET6:
+      GNUNET_assert (addrlen == sizeof (struct sockaddr_in6));
+      v6 = (const struct sockaddr_in6 *) &sa;
+      ok = ( ( (NULL == sh->v6_allowed) ||
+               (check_ipv6_listed (sh->v6_allowed,
+                                   &v6->sin6_addr))) &&
+             ( (NULL == sh->v6_denied) ||
+               (! check_ipv6_listed (sh->v6_denied,
+                                     &v6->sin6_addr)) ) );
+      break;
 #ifndef WINDOWS
-  if (NULL != sctx->addrs)
-    for (i = 0; NULL != sctx->addrs[i]; i++)
-      if ((AF_UNIX == sctx->addrs[i]->sa_family)
-          && ('\0' != ((const struct sockaddr_un *)sctx->addrs[i])->sun_path[0]))
-        GNUNET_DISK_fix_permissions (((const struct sockaddr_un *)sctx->addrs[i])->sun_path,
-                                     sctx->match_uid,
-                                     sctx->match_gid);
+    case AF_UNIX:
+      ok = GNUNET_OK;            /* controlled using file-system ACL now */
+      break;
 #endif
-  sctx->my_handlers = GNUNET_malloc (sizeof (defhandlers));
-  GNUNET_memcpy (sctx->my_handlers, defhandlers, sizeof (defhandlers));
-  i = 0;
-  while ((sctx->my_handlers[i].callback != NULL))
-    sctx->my_handlers[i++].callback_cls = sctx;
-  GNUNET_SERVER_add_handlers (sctx->server, sctx->my_handlers);
-  return sctx;
+    default:
+      LOG (GNUNET_ERROR_TYPE_WARNING,
+           _("Unknown address family %d\n"),
+           sa.ss_family);
+      return;
+    }
+    if (! ok)
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Service rejected incoming connection from %s due to policy.\n",
+           GNUNET_a2s ((const struct sockaddr *) &sa,
+                       addrlen));
+      GNUNET_break (GNUNET_OK ==
+                    GNUNET_NETWORK_socket_close (sock));
+      continue;
+    }
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Service accepted incoming connection from %s.\n",
+         GNUNET_a2s ((const struct sockaddr *) &sa,
+                     addrlen));
+    start_client (slc->sh,
+                  sock);
+  }
+  slc->listen_task
+    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                     slc->listen_socket,
+                                     &accept_client,
+                                     slc);
+}
+
+
+/**
+ * Resume accepting connections from the listen socket.
+ *
+ * @param sh service to resume accepting connections.
+ */
+void
+GNUNET_SERVICE_resume (struct GNUNET_SERVICE_Handle *sh)
+{
+  struct ServiceListenContext *slc;
+
+  for (slc = sh->slc_head; NULL != slc; slc = slc->next)
+  {
+    GNUNET_assert (NULL == slc->listen_task);
+    slc->listen_task
+      = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                       slc->listen_socket,
+                                       &accept_client,
+                                       slc);
+  }
 }
 
 
 /**
- * Obtain the server used by a service.  Note that the server must NOT
- * be destroyed by the caller.
+ * Task run to resume receiving data from the client after
+ * the client called #GNUNET_SERVICE_client_continue().
  *
- * @param ctx the service context returned from the start function
- * @return handle to the server for this service, NULL if there is none
+ * @param cls our `struct GNUNET_SERVICE_Client`
  */
-struct GNUNET_SERVER_Handle *
-GNUNET_SERVICE_get_server (struct GNUNET_SERVICE_Context *ctx)
+static void
+resume_client_receive (void *cls)
 {
-  return ctx->server;
+  struct GNUNET_SERVICE_Client *c = cls;
+  int ret;
+
+  c->recv_task = NULL;
+  /* first, check if there is still something in the buffer */
+  ret = GNUNET_MST_next (c->mst,
+                         GNUNET_YES);
+  if (GNUNET_SYSERR == ret)
+  {
+    if (NULL != c->drop_task)
+      GNUNET_SERVICE_client_drop (c);
+    return;
+  }
+  if (GNUNET_NO == ret)
+    return; /* done processing, wait for more later */
+  GNUNET_assert (GNUNET_OK == ret);
+  if (GNUNET_YES == c->needs_continue)
+    return; /* #GNUNET_MST_next() did give a message to the client */
+  /* need to receive more data from the network first */
+  if (NULL != c->recv_task)
+    return;
+  c->recv_task
+    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
+                                     c->sock,
+                                     &service_client_recv,
+                                     c);
 }
 
 
 /**
- * Get the NULL-terminated array of listen sockets for this service.
+ * Continue receiving further messages from the given client.
+ * Must be called after each message received.
  *
- * @param ctx service context to query
- * @return NULL if there are no listen sockets, otherwise NULL-terminated
- *              array of listen sockets.
+ * @param c the client to continue receiving from
  */
-struct GNUNET_NETWORK_Handle *const*
-GNUNET_SERVICE_get_listen_sockets (struct GNUNET_SERVICE_Context *ctx)
+void
+GNUNET_SERVICE_client_continue (struct GNUNET_SERVICE_Client *c)
 {
-  return ctx->lsocks;
+  GNUNET_assert (GNUNET_YES == c->needs_continue);
+  GNUNET_assert (NULL == c->recv_task);
+  c->needs_continue = GNUNET_NO;
+  if (NULL != c->warn_task)
+  {
+    GNUNET_SCHEDULER_cancel (c->warn_task);
+    c->warn_task = NULL;
+  }
+  c->recv_task
+    = GNUNET_SCHEDULER_add_now (&resume_client_receive,
+                                c);
 }
 
 
 /**
- * Stop a service that was started with "GNUNET_SERVICE_start".
+ * Disable the warning the server issues if a message is not
+ * acknowledged in a timely fashion.  Use this call if a client is
+ * intentionally delayed for a while.  Only applies to the current
+ * message.
  *
- * @param sctx the service context returned from the start function
+ * @param c client for which to disable the warning
  */
 void
-GNUNET_SERVICE_stop (struct GNUNET_SERVICE_Context *sctx)
+GNUNET_SERVICE_client_disable_continue_warning (struct GNUNET_SERVICE_Client *c)
 {
-  unsigned int i;
+  GNUNET_break (NULL != c->warn_task);
+  if (NULL != c->warn_task)
+  {
+    GNUNET_SCHEDULER_cancel (c->warn_task);
+    c->warn_task = NULL;
+  }
+}
 
-#if HAVE_MALLINFO
+
+/**
+ * Asynchronously finish dropping the client.
+ *
+ * @param cls the `struct GNUNET_SERVICE_Client`.
+ */
+static void
+finish_client_drop (void *cls)
+{
+  struct GNUNET_SERVICE_Client *c = cls;
+  struct GNUNET_SERVICE_Handle *sh = c->sh;
+
+  c->drop_task = NULL;
+  GNUNET_assert (NULL == c->send_task);
+  GNUNET_assert (NULL == c->recv_task);
+  GNUNET_assert (NULL == c->warn_task);
+  GNUNET_MST_destroy (c->mst);
+  GNUNET_MQ_destroy (c->mq);
+  if (GNUNET_NO == c->persist)
   {
-    char *counter;
+    GNUNET_break (GNUNET_OK ==
+                  GNUNET_NETWORK_socket_close (c->sock));
+  }
+  else
+  {
+    GNUNET_NETWORK_socket_free_memory_only_ (c->sock);
+  }
+  GNUNET_free (c);
+  if ( (GNUNET_YES == sh->got_shutdown) &&
+       (GNUNET_NO == have_non_monitor_clients (sh)) )
+    GNUNET_SERVICE_shutdown (sh);
+}
 
-    if ( (GNUNET_YES ==
-          GNUNET_CONFIGURATION_have_value (sctx->cfg, sctx->service_name,
-                                           "GAUGER_HEAP")) &&
-         (GNUNET_OK ==
-          GNUNET_CONFIGURATION_get_value_string (sctx->cfg, sctx->service_name,
-                                                 "GAUGER_HEAP",
-                                                 &counter)) )
-    {
-      struct mallinfo mi;
 
-      mi = mallinfo ();
-      GAUGER (sctx->service_name, counter, mi.usmblks, "blocks");
-      GNUNET_free (counter);
-    }
+/**
+ * Ask the server to disconnect from the given client.  This is the
+ * same as returning #GNUNET_SYSERR within the check procedure when
+ * handling a message, wexcept that it allows dropping of a client even
+ * when not handling a message from that client.  The `disconnect_cb`
+ * will be called on @a c even if the application closes the connection
+ * using this function.
+ *
+ * @param c client to disconnect now
+ */
+void
+GNUNET_SERVICE_client_drop (struct GNUNET_SERVICE_Client *c)
+{
+  struct GNUNET_SERVICE_Handle *sh = c->sh;
+
+  if (NULL != c->drop_task)
+  {
+    /* asked to drop twice! */
+    GNUNET_assert (0);
+    return;
   }
-#endif
-  if (NULL != sctx->shutdown_task)
+  GNUNET_CONTAINER_DLL_remove (sh->clients_head,
+                               sh->clients_tail,
+                               c);
+  if (NULL != sh->disconnect_cb)
+    sh->disconnect_cb (sh->cb_cls,
+                       c,
+                       c->user_context);
+  if (NULL != c->warn_task)
   {
-    GNUNET_SCHEDULER_cancel (sctx->shutdown_task);
-    sctx->shutdown_task = NULL;
+    GNUNET_SCHEDULER_cancel (c->warn_task);
+    c->warn_task = NULL;
   }
-  if (NULL != sctx->server)
-    GNUNET_SERVER_destroy (sctx->server);
-  GNUNET_free_non_null (sctx->my_handlers);
-  if (NULL != sctx->addrs)
+  if (NULL != c->recv_task)
   {
-    i = 0;
-    while (NULL != sctx->addrs[i])
-      GNUNET_free (sctx->addrs[i++]);
-    GNUNET_free (sctx->addrs);
-  }
-  GNUNET_free_non_null (sctx->addrlens);
-  GNUNET_free_non_null (sctx->v4_denied);
-  GNUNET_free_non_null (sctx->v6_denied);
-  GNUNET_free_non_null (sctx->v4_allowed);
-  GNUNET_free_non_null (sctx->v6_allowed);
-  GNUNET_free (sctx);
+    GNUNET_SCHEDULER_cancel (c->recv_task);
+    c->recv_task = NULL;
+  }
+  if (NULL != c->send_task)
+  {
+    GNUNET_SCHEDULER_cancel (c->send_task);
+    c->send_task = NULL;
+  }
+  c->drop_task = GNUNET_SCHEDULER_add_now (&finish_client_drop,
+                                           c);
+}
+
+
+/**
+ * Explicitly stops the service.
+ *
+ * @param sh server to shutdown
+ */
+void
+GNUNET_SERVICE_shutdown (struct GNUNET_SERVICE_Handle *sh)
+{
+  struct GNUNET_SERVICE_Client *client;
+
+  GNUNET_SERVICE_suspend (sh);
+  sh->got_shutdown = GNUNET_NO;
+  while (NULL != (client = sh->clients_head))
+    GNUNET_SERVICE_client_drop (client);
+}
+
+
+/**
+ * Set the 'monitor' flag on this client.  Clients which have been
+ * marked as 'monitors' won't prevent the server from shutting down
+ * once #GNUNET_SERVICE_stop_listening() has been invoked.  The idea is
+ * that for "normal" clients we likely want to allow them to process
+ * their requests; however, monitor-clients are likely to 'never'
+ * disconnect during shutdown and thus will not be considered when
+ * determining if the server should continue to exist after
+ * shutdown has been triggered.
+ *
+ * @param c client to mark as a monitor
+ */
+void
+GNUNET_SERVICE_client_mark_monitor (struct GNUNET_SERVICE_Client *c)
+{
+  c->is_monitor = GNUNET_YES;
+  if ( (GNUNET_YES == c->sh->got_shutdown) &&
+       (GNUNET_NO == have_non_monitor_clients (c->sh)) )
+    GNUNET_SERVICE_shutdown (c->sh);
+}
+
+
+/**
+ * Set the persist option on this client.  Indicates that the
+ * underlying socket or fd should never really be closed.  Used for
+ * indicating process death.
+ *
+ * @param c client to persist the socket (never to be closed)
+ */
+void
+GNUNET_SERVICE_client_persist (struct GNUNET_SERVICE_Client *c)
+{
+  c->persist = GNUNET_YES;
+}
+
+
+/**
+ * Obtain the message queue of @a c.  Convenience function.
+ *
+ * @param c the client to continue receiving from
+ * @return the message queue of @a c
+ */
+struct GNUNET_MQ_Handle *
+GNUNET_SERVICE_client_get_mq (struct GNUNET_SERVICE_Client *c)
+{
+  return c->mq;
 }
 
 
-/* end of service.c */
+/* end of service_new.c */
diff --git a/src/util/service_new.c b/src/util/service_new.c
deleted file mode 100644
index 22eec0bde..000000000
--- a/src/util/service_new.c
+++ /dev/null
@@ -1,2615 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-
-/**
- * @file util/service_new.c
- * @brief functions related to starting services (redesign)
- * @author Christian Grothoff
- * @author Florian Dold
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-#include "gnunet_protocols.h"
-#include "gnunet_constants.h"
-#include "gnunet_resolver_service.h"
-#include "speedup.h"
-
-#if HAVE_MALLINFO
-#include <malloc.h>
-#include "gauger.h"
-#endif
-
-
-#define LOG(kind,...) GNUNET_log_from (kind, "util-service", __VA_ARGS__)
-
-#define LOG_STRERROR(kind,syscall) GNUNET_log_from_strerror (kind, "util-service", syscall)
-
-#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util-service", syscall, filename)
-
-
-/**
- * Information the service tracks per listen operation.
- */
-struct ServiceListenContext
-{
-
-  /**
-   * Kept in a DLL.
-   */
-  struct ServiceListenContext *next;
-
-  /**
-   * Kept in a DLL.
-   */
-  struct ServiceListenContext *prev;
-
-  /**
-   * Service this listen context belongs to.
-   */
-  struct GNUNET_SERVICE_Handle *sh;
-
-  /**
-   * Socket we are listening on.
-   */
-  struct GNUNET_NETWORK_Handle *listen_socket;
-
-  /**
-   * Task scheduled to do the listening.
-   */
-  struct GNUNET_SCHEDULER_Task *listen_task;
-
-};
-
-
-/**
- * Handle to a service.
- */
-struct GNUNET_SERVICE_Handle
-{
-  /**
-   * Our configuration.
-   */
-  const struct GNUNET_CONFIGURATION_Handle *cfg;
-
-  /**
-   * Name of our service.
-   */
-  const char *service_name;
-
-  /**
-   * Main service-specific task to run.
-   */
-  GNUNET_SERVICE_InitCallback service_init_cb;
-
-  /**
-   * Function to call when clients connect.
-   */
-  GNUNET_SERVICE_ConnectHandler connect_cb;
-
-  /**
-   * Function to call when clients disconnect / are disconnected.
-   */
-  GNUNET_SERVICE_DisconnectHandler disconnect_cb;
-
-  /**
-   * Closure for @e service_init_cb, @e connect_cb, @e disconnect_cb.
-   */
-  void *cb_cls;
-
-  /**
-   * DLL of listen sockets used to accept new connections.
-   */
-  struct ServiceListenContext *slc_head;
-
-  /**
-   * DLL of listen sockets used to accept new connections.
-   */
-  struct ServiceListenContext *slc_tail;
-
-  /**
-   * Our clients, kept in a DLL.
-   */
-  struct GNUNET_SERVICE_Client *clients_head;
-
-  /**
-   * Our clients, kept in a DLL.
-   */
-  struct GNUNET_SERVICE_Client *clients_tail;
-
-  /**
-   * Message handlers to use for all clients.
-   */
-  struct GNUNET_MQ_MessageHandler *handlers;
-
-  /**
-   * Closure for @e task.
-   */
-  void *task_cls;
-
-  /**
-   * IPv4 addresses that are not allowed to connect.
-   */
-  struct GNUNET_STRINGS_IPv4NetworkPolicy *v4_denied;
-
-  /**
-   * IPv6 addresses that are not allowed to connect.
-   */
-  struct GNUNET_STRINGS_IPv6NetworkPolicy *v6_denied;
-
-  /**
-   * IPv4 addresses that are allowed to connect (if not
-   * set, all are allowed).
-   */
-  struct GNUNET_STRINGS_IPv4NetworkPolicy *v4_allowed;
-
-  /**
-   * IPv6 addresses that are allowed to connect (if not
-   * set, all are allowed).
-   */
-  struct GNUNET_STRINGS_IPv6NetworkPolicy *v6_allowed;
-
-  /**
-   * Do we require a matching UID for UNIX domain socket connections?
-   * #GNUNET_NO means that the UID does not have to match (however,
-   * @e match_gid may still impose other access control checks).
-   */
-  int match_uid;
-
-  /**
-   * Do we require a matching GID for UNIX domain socket connections?
-   * Ignored if @e match_uid is #GNUNET_YES.  Note that this is about
-   * checking that the client's UID is in our group OR that the
-   * client's GID is our GID.  If both "match_gid" and @e match_uid are
-   * #GNUNET_NO, all users on the local system have access.
-   */
-  int match_gid;
-
-  /**
-   * Set to #GNUNET_YES if we got a shutdown signal and terminate
-   * the service if #have_non_monitor_clients() returns #GNUNET_YES.
-   */
-  int got_shutdown;
-
-  /**
-   * Our options.
-   */
-  enum GNUNET_SERVICE_Options options;
-
-  /**
-   * If we are daemonizing, this FD is set to the
-   * pipe to the parent.  Send '.' if we started
-   * ok, '!' if not.  -1 if we are not daemonizing.
-   */
-  int ready_confirm_fd;
-
-  /**
-   * Overall success/failure of the service start.
-   */
-  int ret;
-
-  /**
-   * If #GNUNET_YES, consider unknown message types an error where the
-   * client is disconnected.
-   */
-  int require_found;
-};
-
-
-/**
- * Handle to a client that is connected to a service.
- */
-struct GNUNET_SERVICE_Client
-{
-
-  /**
-   * Kept in a DLL.
-   */
-  struct GNUNET_SERVICE_Client *next;
-
-  /**
-   * Kept in a DLL.
-   */
-  struct GNUNET_SERVICE_Client *prev;
-
-  /**
-   * Service that this client belongs to.
-   */
-  struct GNUNET_SERVICE_Handle *sh;
-
-  /**
-   * Socket of this client.
-   */
-  struct GNUNET_NETWORK_Handle *sock;
-
-  /**
-   * Message queue for the client.
-   */
-  struct GNUNET_MQ_Handle *mq;
-
-  /**
-   * Tokenizer we use for processing incoming data.
-   */
-  struct GNUNET_MessageStreamTokenizer *mst;
-
-  /**
-   * Task that warns about missing calls to
-   * #GNUNET_SERVICE_client_continue().
-   */
-  struct GNUNET_SCHEDULER_Task *warn_task;
-
-  /**
-   * Task run to finish dropping the client after the stack has
-   * properly unwound.
-   */
-  struct GNUNET_SCHEDULER_Task *drop_task;
-
-  /**
-   * Task that receives data from the client to
-   * pass it to the handlers.
-   */
-  struct GNUNET_SCHEDULER_Task *recv_task;
-
-  /**
-   * Task that transmit data to the client.
-   */
-  struct GNUNET_SCHEDULER_Task *send_task;
-
-  /**
-   * Pointer to the message to be transmitted by @e send_task.
-   */
-  const struct GNUNET_MessageHeader *msg;
-
-  /**
-   * User context value, value returned from
-   * the connect callback.
-   */
-  void *user_context;
-
-  /**
-   * Time when we last gave a message from this client
-   * to the application.
-   */
-  struct GNUNET_TIME_Absolute warn_start;
-
-  /**
-   * Current position in @e msg at which we are transmitting.
-   */
-  size_t msg_pos;
-
-  /**
-   * Persist the file handle for this client no matter what happens,
-   * force the OS to close once the process actually dies.  Should only
-   * be used in special cases!
-   */
-  int persist;
-
-  /**
-   * Is this client a 'monitor' client that should not be counted
-   * when deciding on destroying the server during soft shutdown?
-   * (see also #GNUNET_SERVICE_start)
-   */
-  int is_monitor;
-
-  /**
-   * Are we waiting for the application to call #GNUNET_SERVICE_client_continue()?
-   */
-  int needs_continue;
-
-  /**
-   * Type of last message processed (for warn_no_receive_done).
-   */
-  uint16_t warn_type;
-};
-
-
-/**
- * Check if any of the clients we have left are unrelated to
- * monitoring.
- *
- * @param sh service to check clients for
- * @return #GNUNET_YES if we have non-monitoring clients left
- */
-static int
-have_non_monitor_clients (struct GNUNET_SERVICE_Handle *sh)
-{
-  struct GNUNET_SERVICE_Client *client;
-
-  for (client = sh->clients_head;NULL != client; client = client->next)
-  {
-    if (client->is_monitor)
-      continue;
-    return GNUNET_YES;
-  }
-  return GNUNET_NO;
-}
-
-
-/**
- * Shutdown task triggered when a service should be terminated.
- * This considers active clients and the service options to see
- * how this specific service is to be terminated, and depending
- * on this proceeds with the shutdown logic.
- *
- * @param cls our `struct GNUNET_SERVICE_Handle`
- */
-static void
-service_shutdown (void *cls)
-{
-  struct GNUNET_SERVICE_Handle *sh = cls;
-
-  switch (sh->options)
-  {
-  case GNUNET_SERVICE_OPTION_NONE:
-    GNUNET_SERVICE_shutdown (sh);
-    break;
-  case GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN:
-    /* This task should never be run if we are using
-       the manual shutdown. */
-    GNUNET_assert (0);
-    break;
-  case GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN:
-    sh->got_shutdown = GNUNET_YES;
-    GNUNET_SERVICE_suspend (sh);
-    if (GNUNET_NO == have_non_monitor_clients (sh))
-      GNUNET_SERVICE_shutdown (sh);
-    break;
-  }
-}
-
-
-/**
- * First task run by any service.  Initializes our shutdown task,
- * starts the listening operation on our listen sockets and launches
- * the custom logic of the application service.
- *
- * @param cls our `struct GNUNET_SERVICE_Handle`
- */
-static void
-service_main (void *cls)
-{
-  struct GNUNET_SERVICE_Handle *sh = cls;
-
-  if (GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN != sh->options)
-    GNUNET_SCHEDULER_add_shutdown (&service_shutdown,
-                                   sh);
-  GNUNET_SERVICE_resume (sh);
-
-  if (-1 != sh->ready_confirm_fd)
-  {
-    GNUNET_break (1 == WRITE (sh->ready_confirm_fd, ".", 1));
-    GNUNET_break (0 == CLOSE (sh->ready_confirm_fd));
-    sh->ready_confirm_fd = -1;
-  }
-
-  if (NULL != sh->service_init_cb)
-    sh->service_init_cb (sh->cb_cls,
-                         sh->cfg,
-                         sh);
-}
-
-
-/**
- * Parse an IPv4 access control list.
- *
- * @param ret location where to write the ACL (set)
- * @param sh service context to use to get the configuration
- * @param option name of the ACL option to parse
- * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
- *         no ACL configured)
- */
-static int
-process_acl4 (struct GNUNET_STRINGS_IPv4NetworkPolicy **ret,
-              struct GNUNET_SERVICE_Handle *sh,
-              const char *option)
-{
-  char *opt;
-
-  if (! GNUNET_CONFIGURATION_have_value (sh->cfg,
-                                         sh->service_name,
-                                         option))
-  {
-    *ret = NULL;
-    return GNUNET_OK;
-  }
-  GNUNET_break (GNUNET_OK ==
-                GNUNET_CONFIGURATION_get_value_string (sh->cfg,
-                                                       sh->service_name,
-                                                       option,
-                                                       &opt));
-  if (NULL == (*ret = GNUNET_STRINGS_parse_ipv4_policy (opt)))
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         _("Could not parse IPv4 network specification `%s' for `%s:%s'\n"),
-         opt,
-         sh->service_name,
-         option);
-    GNUNET_free (opt);
-    return GNUNET_SYSERR;
-  }
-  GNUNET_free (opt);
-  return GNUNET_OK;
-}
-
-
-/**
- * Parse an IPv6 access control list.
- *
- * @param ret location where to write the ACL (set)
- * @param sh service context to use to get the configuration
- * @param option name of the ACL option to parse
- * @return #GNUNET_SYSERR on parse error, #GNUNET_OK on success (including
- *         no ACL configured)
- */
-static int
-process_acl6 (struct GNUNET_STRINGS_IPv6NetworkPolicy **ret,
-              struct GNUNET_SERVICE_Handle *sh,
-              const char *option)
-{
-  char *opt;
-
-  if (! GNUNET_CONFIGURATION_have_value (sh->cfg,
-                                         sh->service_name,
-                                         option))
-  {
-    *ret = NULL;
-    return GNUNET_OK;
-  }
-  GNUNET_break (GNUNET_OK ==
-                GNUNET_CONFIGURATION_get_value_string (sh->cfg,
-                                                       sh->service_name,
-                                                       option,
-                                                       &opt));
-  if (NULL == (*ret = GNUNET_STRINGS_parse_ipv6_policy (opt)))
-  {
-    LOG (GNUNET_ERROR_TYPE_WARNING,
-         _("Could not parse IPv6 network specification `%s' for `%s:%s'\n"),
-         opt,
-         sh->service_name,
-         option);
-    GNUNET_free (opt);
-    return GNUNET_SYSERR;
-  }
-  GNUNET_free (opt);
-  return GNUNET_OK;
-}
-
-
-/**
- * Add the given UNIX domain path as an address to the
- * list (as the first entry).
- *
- * @param saddrs array to update
- * @param saddrlens where to store the address length
- * @param unixpath path to add
- * @param abstract #GNUNET_YES to add an abstract UNIX domain socket.  This
- *          parameter is ignore on systems other than LINUX
- */
-static void
-add_unixpath (struct sockaddr **saddrs,
-              socklen_t *saddrlens,
-              const char *unixpath,
-              int abstract)
-{
-#ifdef AF_UNIX
-  struct sockaddr_un *un;
-
-  un = GNUNET_new (struct sockaddr_un);
-  un->sun_family = AF_UNIX;
-  strncpy (un->sun_path,
-           unixpath,
-           sizeof (un->sun_path) - 1);
-#ifdef LINUX
-  if (GNUNET_YES == abstract)
-    un->sun_path[0] = '\0';
-#endif
-#if HAVE_SOCKADDR_UN_SUN_LEN
-  un->sun_len = (u_char) sizeof (struct sockaddr_un);
-#endif
-  *saddrs = (struct sockaddr *) un;
-  *saddrlens = sizeof (struct sockaddr_un);
-#else
-  /* this function should never be called
-   * unless AF_UNIX is defined! */
-  GNUNET_assert (0);
-#endif
-}
-
-
-/**
- * Get the list of addresses that a server for the given service
- * should bind to.
- *
- * @param service_name name of the service
- * @param cfg configuration (which specifies the addresses)
- * @param addrs set (call by reference) to an array of pointers to the
- *              addresses the server should bind to and listen on; the
- *              array will be NULL-terminated (on success)
- * @param addr_lens set (call by reference) to an array of the lengths
- *              of the respective `struct sockaddr` struct in the @a addrs
- *              array (on success)
- * @return number of addresses found on success,
- *              #GNUNET_SYSERR if the configuration
- *              did not specify reasonable finding information or
- *              if it specified a hostname that could not be resolved;
- *              #GNUNET_NO if the number of addresses configured is
- *              zero (in this case, `*addrs` and `*addr_lens` will be
- *              set to NULL).
- */
-static int
-get_server_addresses (const char *service_name,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      struct sockaddr ***addrs,
-                      socklen_t **addr_lens)
-{
-  int disablev6;
-  struct GNUNET_NETWORK_Handle *desc;
-  unsigned long long port;
-  char *unixpath;
-  struct addrinfo hints;
-  struct addrinfo *res;
-  struct addrinfo *pos;
-  struct addrinfo *next;
-  unsigned int i;
-  int resi;
-  int ret;
-  int abstract;
-  struct sockaddr **saddrs;
-  socklen_t *saddrlens;
-  char *hostname;
-
-  *addrs = NULL;
-  *addr_lens = NULL;
-  desc = NULL;
-  if (GNUNET_CONFIGURATION_have_value (cfg,
-                                       service_name,
-                                       "DISABLEV6"))
-  {
-    if (GNUNET_SYSERR ==
-        (disablev6 =
-         GNUNET_CONFIGURATION_get_value_yesno (cfg,
-                                               service_name,
-                                               "DISABLEV6")))
-      return GNUNET_SYSERR;
-  }
-  else
-    disablev6 = GNUNET_NO;
-
-  if (! disablev6)
-  {
-    /* probe IPv6 support */
-    desc = GNUNET_NETWORK_socket_create (PF_INET6,
-                                         SOCK_STREAM,
-                                         0);
-    if (NULL == desc)
-    {
-      if ( (ENOBUFS == errno) ||
-           (ENOMEM == errno) ||
-           (ENFILE == errno) ||
-           (EACCES == errno) )
-      {
-        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                      "socket");
-        return GNUNET_SYSERR;
-      }
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Disabling IPv6 support for service `%s', failed to create IPv6 socket: %s\n"),
-           service_name,
-           STRERROR (errno));
-      disablev6 = GNUNET_YES;
-    }
-    else
-    {
-      GNUNET_break (GNUNET_OK ==
-                    GNUNET_NETWORK_socket_close (desc));
-      desc = NULL;
-    }
-  }
-
-  port = 0;
-  if (GNUNET_CONFIGURATION_have_value (cfg,
-                                       service_name,
-                                       "PORT"))
-  {
-    if (GNUNET_OK !=
-        GNUNET_CONFIGURATION_get_value_number (cfg,
-                                               service_name,
-                                               "PORT",
-                                               &port))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Require valid port number for service `%s' in configuration!\n"),
-           service_name);
-    }
-    if (port > 65535)
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Require valid port number for service `%s' in configuration!\n"),
-           service_name);
-      return GNUNET_SYSERR;
-    }
-  }
-
-  if (GNUNET_CONFIGURATION_have_value (cfg,
-                                       service_name,
-                                       "BINDTO"))
-  {
-    GNUNET_break (GNUNET_OK ==
-                  GNUNET_CONFIGURATION_get_value_string (cfg,
-                                                         service_name,
-                                                         "BINDTO",
-                                                         &hostname));
-  }
-  else
-    hostname = NULL;
-
-  unixpath = NULL;
-  abstract = GNUNET_NO;
-#ifdef AF_UNIX
-  if ((GNUNET_YES ==
-       GNUNET_CONFIGURATION_have_value (cfg,
-                                        service_name,
-                                        "UNIXPATH")) &&
-      (GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_filename (cfg,
-                                                service_name,
-                                                "UNIXPATH",
-                                                &unixpath)) &&
-      (0 < strlen (unixpath)))
-  {
-    /* probe UNIX support */
-    struct sockaddr_un s_un;
-
-    if (strlen (unixpath) >= sizeof (s_un.sun_path))
-    {
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           _("UNIXPATH `%s' too long, maximum length is %llu\n"),
-           unixpath,
-           (unsigned long long) sizeof (s_un.sun_path));
-      unixpath = GNUNET_NETWORK_shorten_unixpath (unixpath);
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Using `%s' instead\n"),
-           unixpath);
-    }
-#ifdef LINUX
-    abstract = GNUNET_CONFIGURATION_get_value_yesno (cfg,
-                                                     "TESTING",
-                                                     "USE_ABSTRACT_SOCKETS");
-    if (GNUNET_SYSERR == abstract)
-      abstract = GNUNET_NO;
-#endif
-    if ( (GNUNET_YES != abstract) &&
-         (GNUNET_OK !=
-          GNUNET_DISK_directory_create_for_file (unixpath)) )
-      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
-                                "mkdir",
-                                unixpath);
-  }
-  if (NULL != unixpath)
-  {
-    desc = GNUNET_NETWORK_socket_create (AF_UNIX,
-                                         SOCK_STREAM,
-                                         0);
-    if (NULL == desc)
-    {
-      if ((ENOBUFS == errno) ||
-          (ENOMEM == errno) ||
-          (ENFILE == errno) ||
-          (EACCES == errno))
-      {
-        LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                      "socket");
-        GNUNET_free_non_null (hostname);
-        GNUNET_free (unixpath);
-        return GNUNET_SYSERR;
-      }
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Disabling UNIX domain socket support for service `%s', failed to create UNIX domain socket: %s\n"),
-           service_name,
-           STRERROR (errno));
-      GNUNET_free (unixpath);
-      unixpath = NULL;
-    }
-    else
-    {
-      GNUNET_break (GNUNET_OK ==
-                    GNUNET_NETWORK_socket_close (desc));
-      desc = NULL;
-    }
-  }
-#endif
-
-  if ((0 == port) && (NULL == unixpath))
-  {
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         _("Have neither PORT nor UNIXPATH for service `%s', but one is required\n"),
-         service_name);
-    GNUNET_free_non_null (hostname);
-    return GNUNET_SYSERR;
-  }
-  if (0 == port)
-  {
-    saddrs = GNUNET_new_array (2,
-                               struct sockaddr *);
-    saddrlens = GNUNET_new_array (2,
-                                  socklen_t);
-    add_unixpath (saddrs,
-                  saddrlens,
-                  unixpath,
-                  abstract);
-    GNUNET_free_non_null (unixpath);
-    GNUNET_free_non_null (hostname);
-    *addrs = saddrs;
-    *addr_lens = saddrlens;
-    return 1;
-  }
-
-  if (NULL != hostname)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Resolving `%s' since that is where `%s' will bind to.\n",
-         hostname,
-         service_name);
-    memset (&hints,
-            0,
-            sizeof (struct addrinfo));
-    if (disablev6)
-      hints.ai_family = AF_INET;
-    hints.ai_protocol = IPPROTO_TCP;
-    if ((0 != (ret = getaddrinfo (hostname,
-                                  NULL,
-                                  &hints,
-                                  &res))) ||
-        (NULL == res))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Failed to resolve `%s': %s\n"),
-           hostname,
-           gai_strerror (ret));
-      GNUNET_free (hostname);
-      GNUNET_free_non_null (unixpath);
-      return GNUNET_SYSERR;
-    }
-    next = res;
-    i = 0;
-    while (NULL != (pos = next))
-    {
-      next = pos->ai_next;
-      if ( (disablev6) &&
-           (pos->ai_family == AF_INET6) )
-        continue;
-      i++;
-    }
-    if (0 == i)
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Failed to find %saddress for `%s'.\n"),
-           disablev6 ? "IPv4 " : "",
-           hostname);
-      freeaddrinfo (res);
-      GNUNET_free (hostname);
-      GNUNET_free_non_null (unixpath);
-      return GNUNET_SYSERR;
-    }
-    resi = i;
-    if (NULL != unixpath)
-      resi++;
-    saddrs = GNUNET_new_array (resi + 1,
-                               struct sockaddr *);
-    saddrlens = GNUNET_new_array (resi + 1,
-                                  socklen_t);
-    i = 0;
-    if (NULL != unixpath)
-    {
-      add_unixpath (saddrs,
-                    saddrlens,
-                    unixpath,
-                    abstract);
-      i++;
-    }
-    next = res;
-    while (NULL != (pos = next))
-    {
-      next = pos->ai_next;
-      if ( (disablev6) &&
-           (AF_INET6 == pos->ai_family) )
-        continue;
-      if ( (IPPROTO_TCP != pos->ai_protocol) &&
-           (0 != pos->ai_protocol) )
-        continue;               /* not TCP */
-      if ( (SOCK_STREAM != pos->ai_socktype) &&
-           (0 != pos->ai_socktype) )
-        continue;               /* huh? */
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Service `%s' will bind to `%s'\n",
-           service_name,
-           GNUNET_a2s (pos->ai_addr,
-                       pos->ai_addrlen));
-      if (AF_INET == pos->ai_family)
-      {
-        GNUNET_assert (sizeof (struct sockaddr_in) == pos->ai_addrlen);
-        saddrlens[i] = pos->ai_addrlen;
-        saddrs[i] = GNUNET_malloc (saddrlens[i]);
-        GNUNET_memcpy (saddrs[i],
-                       pos->ai_addr,
-                       saddrlens[i]);
-        ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
-      }
-      else
-      {
-        GNUNET_assert (AF_INET6 == pos->ai_family);
-        GNUNET_assert (sizeof (struct sockaddr_in6) == pos->ai_addrlen);
-        saddrlens[i] = pos->ai_addrlen;
-        saddrs[i] = GNUNET_malloc (saddrlens[i]);
-        GNUNET_memcpy (saddrs[i],
-                       pos->ai_addr,
-                       saddrlens[i]);
-        ((struct sockaddr_in6 *) saddrs[i])->sin6_port = htons (port);
-      }
-      i++;
-    }
-    GNUNET_free (hostname);
-    freeaddrinfo (res);
-    resi = i;
-  }
-  else
-  {
-    /* will bind against everything, just set port */
-    if (disablev6)
-    {
-      /* V4-only */
-      resi = 1;
-      if (NULL != unixpath)
-        resi++;
-      i = 0;
-      saddrs = GNUNET_new_array (resi + 1,
-                                 struct sockaddr *);
-      saddrlens = GNUNET_new_array (resi + 1,
-                                    socklen_t);
-      if (NULL != unixpath)
-      {
-        add_unixpath (saddrs,
-                      saddrlens,
-                      unixpath,
-                      abstract);
-        i++;
-      }
-      saddrlens[i] = sizeof (struct sockaddr_in);
-      saddrs[i] = GNUNET_malloc (saddrlens[i]);
-#if HAVE_SOCKADDR_IN_SIN_LEN
-      ((struct sockaddr_in *) saddrs[i])->sin_len = saddrlens[i];
-#endif
-      ((struct sockaddr_in *) saddrs[i])->sin_family = AF_INET;
-      ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
-    }
-    else
-    {
-      /* dual stack */
-      resi = 2;
-      if (NULL != unixpath)
-        resi++;
-      saddrs = GNUNET_new_array (resi + 1,
-                                 struct sockaddr *);
-      saddrlens = GNUNET_new_array (resi + 1,
-                                    socklen_t);
-      i = 0;
-      if (NULL != unixpath)
-      {
-        add_unixpath (saddrs,
-                      saddrlens,
-                      unixpath,
-                      abstract);
-        i++;
-      }
-      saddrlens[i] = sizeof (struct sockaddr_in6);
-      saddrs[i] = GNUNET_malloc (saddrlens[i]);
-#if HAVE_SOCKADDR_IN_SIN_LEN
-      ((struct sockaddr_in6 *) saddrs[i])->sin6_len = saddrlens[0];
-#endif
-      ((struct sockaddr_in6 *) saddrs[i])->sin6_family = AF_INET6;
-      ((struct sockaddr_in6 *) saddrs[i])->sin6_port = htons (port);
-      i++;
-      saddrlens[i] = sizeof (struct sockaddr_in);
-      saddrs[i] = GNUNET_malloc (saddrlens[i]);
-#if HAVE_SOCKADDR_IN_SIN_LEN
-      ((struct sockaddr_in *) saddrs[i])->sin_len = saddrlens[1];
-#endif
-      ((struct sockaddr_in *) saddrs[i])->sin_family = AF_INET;
-      ((struct sockaddr_in *) saddrs[i])->sin_port = htons (port);
-    }
-  }
-  GNUNET_free_non_null (unixpath);
-  *addrs = saddrs;
-  *addr_lens = saddrlens;
-  return resi;
-}
-
-
-#ifdef MINGW
-/**
- * Read listen sockets from the parent process (ARM).
- *
- * @param sh service context to initialize
- * @return NULL-terminated array of sockets on success,
- *         NULL if not ok (must bind yourself)
- */
-static struct GNUNET_NETWORK_Handle **
-receive_sockets_from_parent (struct GNUNET_SERVICE_Handle *sh)
-{
-  static struct GNUNET_NETWORK_Handle **lsocks;
-  const char *env_buf;
-  int fail;
-  uint64_t count;
-  uint64_t i;
-  HANDLE lsocks_pipe;
-
-  env_buf = getenv ("GNUNET_OS_READ_LSOCKS");
-  if ( (NULL == env_buf) ||
-       (strlen (env_buf) <= 0) )
-    return NULL;
-  /* Using W32 API directly here, because this pipe will
-   * never be used outside of this function, and it's just too much of a bother
-   * to create a GNUnet API that boxes a HANDLE (the way it is done with socks)
-   */
-  lsocks_pipe = (HANDLE) strtoul (env_buf,
-                                  NULL,
-                                  10);
-  if ( (0 == lsocks_pipe) ||
-       (INVALID_HANDLE_VALUE == lsocks_pipe))
-    return NULL;
-  fail = 1;
-  do
-  {
-    int ret;
-    int fail2;
-    DWORD rd;
-
-    ret = ReadFile (lsocks_pipe,
-                    &count,
-                    sizeof (count),
-                    &rd,
-                    NULL);
-    if ( (0 == ret) ||
-         (sizeof (count) != rd) ||
-         (0 == count) )
-      break;
-    lsocks = GNUNET_new_array (count + 1,
-                               struct GNUNET_NETWORK_Handle *);
-
-    fail2 = 1;
-    for (i = 0; i < count; i++)
-    {
-      WSAPROTOCOL_INFOA pi;
-      uint64_t size;
-      SOCKET s;
-
-      ret = ReadFile (lsocks_pipe,
-                      &size,
-                      sizeof (size),
-                      &rd,
-                      NULL);
-      if ( (0 == ret) ||
-           (sizeof (size) != rd) ||
-           (sizeof (pi) != size) )
-        break;
-      ret = ReadFile (lsocks_pipe,
-                      &pi,
-                      sizeof (pi),
-                      &rd,
-                      NULL);
-      if ( (0 == ret) ||
-           (sizeof (pi) != rd))
-        break;
-      s = WSASocketA (pi.iAddressFamily,
-                      pi.iSocketType,
-                      pi.iProtocol,
-                      &pi,
-                      0,
-                      WSA_FLAG_OVERLAPPED);
-      lsocks[i] = GNUNET_NETWORK_socket_box_native (s);
-      if (NULL == lsocks[i])
-        break;
-      else if (i == count - 1)
-        fail2 = 0;
-    }
-    if (fail2)
-      break;
-    lsocks[count] = NULL;
-    fail = 0;
-  }
-  while (fail);
-  CloseHandle (lsocks_pipe);
-
-  if (fail)
-  {
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         _("Could not access a pre-bound socket, will try to bind myself\n"));
-    for (i = 0; (i < count) && (NULL != lsocks[i]); i++)
-      GNUNET_break (GNUNET_OK ==
-                    GNUNET_NETWORK_socket_close (lsocks[i]));
-    GNUNET_free (lsocks);
-    return NULL;
-  }
-  return lsocks;
-}
-#endif
-
-
-/**
- * Create and initialize a listen socket for the server.
- *
- * @param server_addr address to listen on
- * @param socklen length of @a server_addr
- * @return NULL on error, otherwise the listen socket
- */
-static struct GNUNET_NETWORK_Handle *
-open_listen_socket (const struct sockaddr *server_addr,
-                    socklen_t socklen)
-{
-  struct GNUNET_NETWORK_Handle *sock;
-  uint16_t port;
-  int eno;
-
-  switch (server_addr->sa_family)
-  {
-  case AF_INET:
-    port = ntohs (((const struct sockaddr_in *) server_addr)->sin_port);
-    break;
-  case AF_INET6:
-    port = ntohs (((const struct sockaddr_in6 *) server_addr)->sin6_port);
-    break;
-  case AF_UNIX:
-    port = 0;
-    break;
-  default:
-    GNUNET_break (0);
-    port = 0;
-    break;
-  }
-  sock = GNUNET_NETWORK_socket_create (server_addr->sa_family,
-                                       SOCK_STREAM,
-                                       0);
-  if (NULL == sock)
-  {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "socket");
-    errno = 0;
-    return NULL;
-  }
-  /* bind the socket */
-  if (GNUNET_OK != GNUNET_NETWORK_socket_bind (sock,
-                                               server_addr,
-                                               socklen))
-  {
-    eno = errno;
-    if (EADDRINUSE != errno)
-    {
-      /* we don't log 'EADDRINUSE' here since an IPv4 bind may
-       * fail if we already took the port on IPv6; if both IPv4 and
-       * IPv6 binds fail, then our caller will log using the
-       * errno preserved in 'eno' */
-      LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                    "bind");
-      if (0 != port)
-        LOG (GNUNET_ERROR_TYPE_ERROR,
-             _("`%s' failed for port %d (%s).\n"),
-             "bind",
-             port,
-             (AF_INET == server_addr->sa_family) ? "IPv4" : "IPv6");
-      eno = 0;
-    }
-    else
-    {
-      if (0 != port)
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             _("`%s' failed for port %d (%s): address already in use\n"),
-             "bind", port,
-             (AF_INET == server_addr->sa_family) ? "IPv4" : "IPv6");
-      else if (AF_UNIX == server_addr->sa_family)
-      {
-        LOG (GNUNET_ERROR_TYPE_WARNING,
-             _("`%s' failed for `%s': address already in use\n"),
-             "bind",
-             GNUNET_a2s (server_addr, socklen));
-      }
-    }
-    GNUNET_break (GNUNET_OK ==
-                  GNUNET_NETWORK_socket_close (sock));
-    errno = eno;
-    return NULL;
-  }
-  if (GNUNET_OK != GNUNET_NETWORK_socket_listen (sock,
-                                                 5))
-  {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "listen");
-    GNUNET_break (GNUNET_OK ==
-                  GNUNET_NETWORK_socket_close (sock));
-    errno = 0;
-    return NULL;
-  }
-  if (0 != port)
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Server starts to listen on port %u.\n",
-         port);
-  return sock;
-}
-
-
-/**
- * Setup service handle
- *
- * Configuration may specify:
- * - PORT (where to bind to for TCP)
- * - UNIXPATH (where to bind to for UNIX domain sockets)
- * - DISABLEV6 (disable support for IPv6, otherwise we use dual-stack)
- * - BINDTO (hostname or IP address to bind to, otherwise we take everything)
- * - ACCEPT_FROM  (only allow connections from specified IPv4 subnets)
- * - ACCEPT_FROM6 (only allow connections from specified IPv6 subnets)
- * - REJECT_FROM  (disallow allow connections from specified IPv4 subnets)
- * - REJECT_FROM6 (disallow allow connections from specified IPv6 subnets)
- *
- * @param sh service context to initialize
- * @return #GNUNET_OK if configuration succeeded
- */
-static int
-setup_service (struct GNUNET_SERVICE_Handle *sh)
-{
-  int tolerant;
-  struct GNUNET_NETWORK_Handle **lsocks;
-#ifndef MINGW
-  const char *nfds;
-  unsigned int cnt;
-  int flags;
-#endif
-
-  if (GNUNET_CONFIGURATION_have_value
-      (sh->cfg,
-       sh->service_name,
-       "TOLERANT"))
-  {
-    if (GNUNET_SYSERR ==
-        (tolerant =
-         GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
-                                               sh->service_name,
-                                               "TOLERANT")))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Specified value for `%s' of service `%s' is invalid\n"),
-           "TOLERANT",
-           sh->service_name);
-      return GNUNET_SYSERR;
-    }
-  }
-  else
-    tolerant = GNUNET_NO;
-
-  lsocks = NULL;
-#ifndef MINGW
-  errno = 0;
-  if ( (NULL != (nfds = getenv ("LISTEN_FDS"))) &&
-       (1 == SSCANF (nfds,
-                     "%u",
-                     &cnt)) &&
-       (cnt > 0) &&
-       (cnt < FD_SETSIZE) &&
-       (cnt + 4 < FD_SETSIZE) )
-  {
-    lsocks = GNUNET_new_array (cnt + 1,
-                               struct GNUNET_NETWORK_Handle *);
-    while (0 < cnt--)
-    {
-      flags = fcntl (3 + cnt,
-                     F_GETFD);
-      if ( (flags < 0) ||
-           (0 != (flags & FD_CLOEXEC)) ||
-           (NULL ==
-            (lsocks[cnt] = GNUNET_NETWORK_socket_box_native (3 + cnt))))
-      {
-        LOG (GNUNET_ERROR_TYPE_ERROR,
-             _("Could not access pre-bound socket %u, will try to bind myself\n"),
-             (unsigned int) 3 + cnt);
-        cnt++;
-        while (NULL != lsocks[cnt])
-          GNUNET_break (GNUNET_OK ==
-                        GNUNET_NETWORK_socket_close (lsocks[cnt++]));
-        GNUNET_free (lsocks);
-        lsocks = NULL;
-        break;
-      }
-    }
-    unsetenv ("LISTEN_FDS");
-  }
-#else
-  if (NULL != getenv ("GNUNET_OS_READ_LSOCKS"))
-  {
-    lsocks = receive_sockets_from_parent (sh);
-    putenv ("GNUNET_OS_READ_LSOCKS=");
-  }
-#endif
-
-  if (NULL != lsocks)
-  {
-    /* listen only on inherited sockets if we have any */
-    struct GNUNET_NETWORK_Handle **ls;
-
-    for (ls = lsocks; NULL != *ls; ls++)
-    {
-      struct ServiceListenContext *slc;
-
-      slc = GNUNET_new (struct ServiceListenContext);
-      slc->sh = sh;
-      slc->listen_socket = *ls;
-      GNUNET_CONTAINER_DLL_insert (sh->slc_head,
-                                   sh->slc_tail,
-                                   slc);
-    }
-    GNUNET_free (lsocks);
-  }
-  else
-  {
-    struct sockaddr **addrs;
-    socklen_t *addrlens;
-    int num;
-
-    num = get_server_addresses (sh->service_name,
-                                sh->cfg,
-                                &addrs,
-                                &addrlens);
-    if (GNUNET_SYSERR == num)
-      return GNUNET_SYSERR;
-
-    for (int i = 0; i < num; i++)
-    {
-      struct ServiceListenContext *slc;
-
-      slc = GNUNET_new (struct ServiceListenContext);
-      slc->sh = sh;
-      slc->listen_socket = open_listen_socket (addrs[i],
-                                               addrlens[i]);
-      if (NULL == slc->listen_socket)
-      {
-        GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
-                             "bind");
-        GNUNET_free (addrs[i++]);
-        GNUNET_free (slc);
-        continue;
-      }
-      GNUNET_free (addrs[i++]);
-      GNUNET_CONTAINER_DLL_insert (sh->slc_head,
-                                   sh->slc_tail,
-                                   slc);
-    }
-    GNUNET_free_non_null (addrlens);
-    GNUNET_free_non_null (addrs);
-    if ( (0 != num) &&
-         (NULL == sh->slc_head) )
-    {
-      /* All attempts to bind failed, hard failure */
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  _("Could not bind to any of the ports I was supposed to, refusing to run!\n"));
-      return GNUNET_SYSERR;
-    }
-  }
-
-  sh->require_found = tolerant ? GNUNET_NO : GNUNET_YES;
-  sh->match_uid
-    = GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
-                                            sh->service_name,
-                                            "UNIX_MATCH_UID");
-  sh->match_gid
-    = GNUNET_CONFIGURATION_get_value_yesno (sh->cfg,
-                                            sh->service_name,
-                                            "UNIX_MATCH_GID");
-  process_acl4 (&sh->v4_denied,
-                sh,
-                "REJECT_FROM");
-  process_acl4 (&sh->v4_allowed,
-                sh,
-                "ACCEPT_FROM");
-  process_acl6 (&sh->v6_denied,
-                sh,
-                "REJECT_FROM6");
-  process_acl6 (&sh->v6_allowed,
-                sh,
-                "ACCEPT_FROM6");
-  return GNUNET_OK;
-}
-
-
-/**
- * Get the name of the user that'll be used
- * to provide the service.
- *
- * @param sh service context
- * @return value of the 'USERNAME' option
- */
-static char *
-get_user_name (struct GNUNET_SERVICE_Handle *sh)
-{
-  char *un;
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (sh->cfg,
-                                               sh->service_name,
-                                               "USERNAME",
-                                               &un))
-    return NULL;
-  return un;
-}
-
-
-/**
- * Set user ID.
- *
- * @param sh service context
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-static int
-set_user_id (struct GNUNET_SERVICE_Handle *sh)
-{
-  char *user;
-
-  if (NULL == (user = get_user_name (sh)))
-    return GNUNET_OK;           /* keep */
-#ifndef MINGW
-  struct passwd *pws;
-
-  errno = 0;
-  pws = getpwnam (user);
-  if (NULL == pws)
-  {
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         _("Cannot obtain information about user `%s': %s\n"),
-         user,
-         errno == 0 ? _("No such user") : STRERROR (errno));
-    GNUNET_free (user);
-    return GNUNET_SYSERR;
-  }
-  if ( (0 != setgid (pws->pw_gid)) ||
-       (0 != setegid (pws->pw_gid)) ||
-#if HAVE_INITGROUPS
-       (0 != initgroups (user,
-                         pws->pw_gid)) ||
-#endif
-       (0 != setuid (pws->pw_uid)) ||
-       (0 != seteuid (pws->pw_uid)))
-  {
-    if ((0 != setregid (pws->pw_gid,
-                        pws->pw_gid)) ||
-        (0 != setreuid (pws->pw_uid,
-                        pws->pw_uid)))
-    {
-      LOG (GNUNET_ERROR_TYPE_ERROR,
-           _("Cannot change user/group to `%s': %s\n"),
-           user,
-           STRERROR (errno));
-      GNUNET_free (user);
-      return GNUNET_SYSERR;
-    }
-  }
-#endif
-  GNUNET_free (user);
-  return GNUNET_OK;
-}
-
-
-/**
- * Get the name of the file where we will
- * write the PID of the service.
- *
- * @param sh service context
- * @return name of the file for the process ID
- */
-static char *
-get_pid_file_name (struct GNUNET_SERVICE_Handle *sh)
-{
-  char *pif;
-
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_filename (sh->cfg,
-                                               sh->service_name,
-                                               "PIDFILE",
-                                               &pif))
-    return NULL;
-  return pif;
-}
-
-
-/**
- * Delete the PID file that was created by our parent.
- *
- * @param sh service context
- */
-static void
-pid_file_delete (struct GNUNET_SERVICE_Handle *sh)
-{
-  char *pif = get_pid_file_name (sh);
-
-  if (NULL == pif)
-    return;                     /* no PID file */
-  if (0 != UNLINK (pif))
-    LOG_STRERROR_FILE (GNUNET_ERROR_TYPE_WARNING,
-                       "unlink",
-                       pif);
-  GNUNET_free (pif);
-}
-
-
-/**
- * Detach from terminal.
- *
- * @param sh service context
- * @return #GNUNET_OK on success, #GNUNET_SYSERR on error
- */
-static int
-detach_terminal (struct GNUNET_SERVICE_Handle *sh)
-{
-#ifndef MINGW
-  pid_t pid;
-  int nullfd;
-  int filedes[2];
-
-  if (0 != PIPE (filedes))
-  {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "pipe");
-    return GNUNET_SYSERR;
-  }
-  pid = fork ();
-  if (pid < 0)
-  {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "fork");
-    return GNUNET_SYSERR;
-  }
-  if (0 != pid)
-  {
-    /* Parent */
-    char c;
-
-    GNUNET_break (0 == CLOSE (filedes[1]));
-    c = 'X';
-    if (1 != READ (filedes[0],
-                   &c,
-                   sizeof (char)))
-      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING,
-                    "read");
-    fflush (stdout);
-    switch (c)
-    {
-    case '.':
-      exit (0);
-    case 'I':
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Service process failed to initialize\n"));
-      break;
-    case 'S':
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Service process could not initialize server function\n"));
-      break;
-    case 'X':
-      LOG (GNUNET_ERROR_TYPE_INFO,
-           _("Service process failed to report status\n"));
-      break;
-    }
-    exit (1);                   /* child reported error */
-  }
-  GNUNET_break (0 == CLOSE (0));
-  GNUNET_break (0 == CLOSE (1));
-  GNUNET_break (0 == CLOSE (filedes[0]));
-  nullfd = OPEN ("/dev/null",
-                 O_RDWR | O_APPEND);
-  if (nullfd < 0)
-    return GNUNET_SYSERR;
-  /* set stdin/stdout to /dev/null */
-  if ( (dup2 (nullfd, 0) < 0) ||
-       (dup2 (nullfd, 1) < 0) )
-  {
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "dup2");
-    (void) CLOSE (nullfd);
-    return GNUNET_SYSERR;
-  }
-  (void) CLOSE (nullfd);
-  /* Detach from controlling terminal */
-  pid = setsid ();
-  if (-1 == pid)
-    LOG_STRERROR (GNUNET_ERROR_TYPE_ERROR,
-                  "setsid");
-  sh->ready_confirm_fd = filedes[1];
-#else
-  /* FIXME: we probably need to do something else
-   * elsewhere in order to fork the process itself... */
-  FreeConsole ();
-#endif
-  return GNUNET_OK;
-}
-
-
-/**
- * Tear down the service, closing the listen sockets and
- * freeing the ACLs.
- *
- * @param sh handle to the service to tear down.
- */
-static void
-teardown_service (struct GNUNET_SERVICE_Handle *sh)
-{
-  struct ServiceListenContext *slc;
-
-  GNUNET_free_non_null (sh->v4_denied);
-  GNUNET_free_non_null (sh->v6_denied);
-  GNUNET_free_non_null (sh->v4_allowed);
-  GNUNET_free_non_null (sh->v6_allowed);
-  while (NULL != (slc = sh->slc_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (sh->slc_head,
-                                 sh->slc_tail,
-                                 slc);
-    if (NULL != slc->listen_task)
-      GNUNET_SCHEDULER_cancel (slc->listen_task);
-    GNUNET_break (GNUNET_OK ==
-                  GNUNET_NETWORK_socket_close (slc->listen_socket));
-    GNUNET_free (slc);
-  }
-}
-
-
-/**
- * Low-level function to start a service if the scheduler
- * is already running.  Should only be used directly in
- * special cases.
- *
- * The function will launch the service with the name @a service_name
- * using the @a service_options to configure its shutdown
- * behavior. When clients connect or disconnect, the respective
- * @a connect_cb or @a disconnect_cb functions will be called. For
- * messages received from the clients, the respective @a handlers will
- * be invoked; for the closure of the handlers we use the return value
- * from the @a connect_cb invocation of the respective client.
- *
- * Each handler MUST call #GNUNET_SERVICE_client_continue() after each
- * message to receive further messages from this client.  If
- * #GNUNET_SERVICE_client_continue() is not called within a short
- * time, a warning will be logged. If delays are expected, services
- * should call #GNUNET_SERVICE_client_disable_continue_warning() to
- * disable the warning.
- *
- * Clients sending invalid messages (based on @a handlers) will be
- * dropped. Additionally, clients can be dropped at any time using
- * #GNUNET_SERVICE_client_drop().
- *
- * The service must be stopped using #GNUNET_SERVICE_stoP().
- *
- * @param service_name name of the service to run
- * @param cfg configuration to use
- * @param connect_cb function to call whenever a client connects
- * @param disconnect_cb function to call whenever a client disconnects
- * @param cls closure argument for @a connect_cb and @a disconnect_cb
- * @param handlers NULL-terminated array of message handlers for the service,
- *                 the closure will be set to the value returned by
- *                 the @a connect_cb for the respective connection
- * @return NULL on error
- */
-struct GNUNET_SERVICE_Handle *
-GNUNET_SERVICE_starT (const char *service_name,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      GNUNET_SERVICE_ConnectHandler connect_cb,
-                      GNUNET_SERVICE_DisconnectHandler disconnect_cb,
-                      void *cls,
-                      const struct GNUNET_MQ_MessageHandler *handlers)
-{
-  struct GNUNET_SERVICE_Handle *sh;
-
-  sh = GNUNET_new (struct GNUNET_SERVICE_Handle);
-  sh->service_name = service_name;
-  sh->cfg = cfg;
-  sh->connect_cb = connect_cb;
-  sh->disconnect_cb = disconnect_cb;
-  sh->cb_cls = cls;
-  sh->handlers = GNUNET_MQ_copy_handlers (handlers);
-  if (GNUNET_OK != setup_service (sh))
-  {
-    GNUNET_free_non_null (sh->handlers);
-    GNUNET_free (sh);
-    return NULL;
-  }
-  GNUNET_SERVICE_resume (sh);
-  return sh;
-}
-
-
-/**
- * Stops a service that was started with #GNUNET_SERVICE_starT().
- *
- * @param srv service to stop
- */
-void
-GNUNET_SERVICE_stoP (struct GNUNET_SERVICE_Handle *srv)
-{
-  struct GNUNET_SERVICE_Client *client;
-
-  GNUNET_SERVICE_suspend (srv);
-  while (NULL != (client = srv->clients_head))
-    GNUNET_SERVICE_client_drop (client);
-  teardown_service (srv);
-  GNUNET_free_non_null (srv->handlers);
-  GNUNET_free (srv);
-}
-
-
-/**
- * Creates the "main" function for a GNUnet service.  You
- * should almost always use the #GNUNET_SERVICE_MAIN macro
- * instead of calling this function directly (except
- * for ARM, which should call this function directly).
- *
- * The function will launch the service with the name @a service_name
- * using the @a service_options to configure its shutdown
- * behavior. Once the service is ready, the @a init_cb will be called
- * for service-specific initialization.  @a init_cb will be given the
- * service handler which can be used to control the service's
- * availability.  When clients connect or disconnect, the respective
- * @a connect_cb or @a disconnect_cb functions will be called. For
- * messages received from the clients, the respective @a handlers will
- * be invoked; for the closure of the handlers we use the return value
- * from the @a connect_cb invocation of the respective client.
- *
- * Each handler MUST call #GNUNET_SERVICE_client_continue() after each
- * message to receive further messages from this client.  If
- * #GNUNET_SERVICE_client_continue() is not called within a short
- * time, a warning will be logged. If delays are expected, services
- * should call #GNUNET_SERVICE_client_disable_continue_warning() to
- * disable the warning.
- *
- * Clients sending invalid messages (based on @a handlers) will be
- * dropped. Additionally, clients can be dropped at any time using
- * #GNUNET_SERVICE_client_drop().
- *
- * @param argc number of command-line arguments in @a argv
- * @param argv array of command-line arguments
- * @param service_name name of the service to run
- * @param options options controlling shutdown of the service
- * @param service_init_cb function to call once the service is ready
- * @param connect_cb function to call whenever a client connects
- * @param disconnect_cb function to call whenever a client disconnects
- * @param cls closure argument for @a service_init_cb, @a connect_cb and @a disconnect_cb
- * @param handlers NULL-terminated array of message handlers for the service,
- *                 the closure will be set to the value returned by
- *                 the @a connect_cb for the respective connection
- * @return 0 on success, non-zero on error
- */
-int
-GNUNET_SERVICE_ruN_ (int argc,
-                     char *const *argv,
-                     const char *service_name,
-                     enum GNUNET_SERVICE_Options options,
-                     GNUNET_SERVICE_InitCallback service_init_cb,
-                     GNUNET_SERVICE_ConnectHandler connect_cb,
-                     GNUNET_SERVICE_DisconnectHandler disconnect_cb,
-                     void *cls,
-                     const struct GNUNET_MQ_MessageHandler *handlers)
-{
-  struct GNUNET_SERVICE_Handle sh;
-  char *cfg_filename;
-  char *opt_cfg_filename;
-  char *loglev;
-  const char *xdg;
-  char *logfile;
-  int do_daemonize;
-  unsigned long long skew_offset;
-  unsigned long long skew_variance;
-  long long clock_offset;
-  struct GNUNET_CONFIGURATION_Handle *cfg;
-  int ret;
-  int err;
-
-  struct GNUNET_GETOPT_CommandLineOption service_options[] = {
-    GNUNET_GETOPT_OPTION_CFG_FILE (&opt_cfg_filename),
-    {'d', "daemonize", NULL,
-     gettext_noop ("do daemonize (detach from terminal)"), 0,
-     GNUNET_GETOPT_set_one, &do_daemonize},
-    GNUNET_GETOPT_OPTION_HELP (NULL),
-    GNUNET_GETOPT_OPTION_LOGLEVEL (&loglev),
-    GNUNET_GETOPT_OPTION_LOGFILE (&logfile),
-    GNUNET_GETOPT_OPTION_VERSION (PACKAGE_VERSION " " VCS_VERSION),
-    GNUNET_GETOPT_OPTION_END
-  };
-
-  memset (&sh,
-          0,
-          sizeof (sh));
-  xdg = getenv ("XDG_CONFIG_HOME");
-  if (NULL != xdg)
-    GNUNET_asprintf (&cfg_filename,
-                     "%s%s%s",
-                     xdg,
-                     DIR_SEPARATOR_STR,
-                     GNUNET_OS_project_data_get ()->config_file);
-  else
-    cfg_filename = GNUNET_strdup (GNUNET_OS_project_data_get ()->user_config_file);
-  sh.ready_confirm_fd = -1;
-  sh.options = options;
-  sh.cfg = cfg = GNUNET_CONFIGURATION_create ();
-  sh.service_init_cb = service_init_cb;
-  sh.connect_cb = connect_cb;
-  sh.disconnect_cb = disconnect_cb;
-  sh.cb_cls = cls;
-  sh.handlers = GNUNET_MQ_copy_handlers (handlers);
-  sh.service_name = service_name;
-
-  /* setup subsystems */
-  loglev = NULL;
-  logfile = NULL;
-  opt_cfg_filename = NULL;
-  do_daemonize = 0;
-  ret = GNUNET_GETOPT_run (service_name,
-                           service_options,
-                           argc,
-                           argv);
-  if (GNUNET_SYSERR == ret)
-    goto shutdown;
-  if (GNUNET_NO == ret)
-  {
-    err = 0;
-    goto shutdown;
-  }
-  if (GNUNET_OK != GNUNET_log_setup (service_name,
-                                     loglev,
-                                     logfile))
-  {
-    GNUNET_break (0);
-    goto shutdown;
-  }
-  if (NULL == opt_cfg_filename)
-    opt_cfg_filename = GNUNET_strdup (cfg_filename);
-  if (GNUNET_YES == GNUNET_DISK_file_test (opt_cfg_filename))
-  {
-    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg,
-                                                    opt_cfg_filename))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  _("Malformed configuration file `%s', exit ...\n"),
-                  opt_cfg_filename);
-      goto shutdown;
-    }
-  }
-  else
-  {
-    if (GNUNET_SYSERR == GNUNET_CONFIGURATION_load (cfg,
-                                                    NULL))
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  _("Malformed configuration, exit ...\n"));
-      goto shutdown;
-    }
-    if (0 != strcmp (opt_cfg_filename,
-                     cfg_filename))
-      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                  _("Could not access configuration file `%s'\n"),
-                  opt_cfg_filename);
-  }
-  if (GNUNET_OK != setup_service (&sh))
-    goto shutdown;
-  if ( (1 == do_daemonize) &&
-       (GNUNET_OK != detach_terminal (&sh)) )
-  {
-    GNUNET_break (0);
-    goto shutdown;
-  }
-  if (GNUNET_OK != set_user_id (&sh))
-    goto shutdown;
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Service `%s' runs with configuration from `%s'\n",
-       service_name,
-       opt_cfg_filename);
-  if ((GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_number (sh.cfg,
-                                              "TESTING",
-                                              "SKEW_OFFSET",
-                                              &skew_offset)) &&
-      (GNUNET_OK ==
-       GNUNET_CONFIGURATION_get_value_number (sh.cfg,
-                                              "TESTING",
-                                              "SKEW_VARIANCE",
-                                              &skew_variance)))
-  {
-    clock_offset = skew_offset - skew_variance;
-    GNUNET_TIME_set_offset (clock_offset);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Skewing clock by %dll ms\n",
-         clock_offset);
-  }
-  GNUNET_RESOLVER_connect (sh.cfg);
-
-  /* actually run service */
-  err = 0;
-  GNUNET_SCHEDULER_run (&service_main,
-                        &sh);
-  /* shutdown */
-  if (1 == do_daemonize)
-    pid_file_delete (&sh);
-
-shutdown:
-  if (-1 != sh.ready_confirm_fd)
-  {
-    if (1 != WRITE (sh.ready_confirm_fd,
-                    err ? "I" : "S",
-                    1))
-      LOG_STRERROR (GNUNET_ERROR_TYPE_WARNING,
-                    "write");
-    GNUNET_break (0 == CLOSE (sh.ready_confirm_fd));
-  }
-#if HAVE_MALLINFO
-  {
-    char *counter;
-
-    if ( (GNUNET_YES ==
-          GNUNET_CONFIGURATION_have_value (sh.cfg,
-                                           service_name,
-                                           "GAUGER_HEAP")) &&
-         (GNUNET_OK ==
-          GNUNET_CONFIGURATION_get_value_string (sh.cfg,
-                                                 service_name,
-                                                 "GAUGER_HEAP",
-                                                 &counter)) )
-    {
-      struct mallinfo mi;
-
-      mi = mallinfo ();
-      GAUGER (service_name,
-              counter,
-              mi.usmblks,
-              "blocks");
-      GNUNET_free (counter);
-    }
-  }
-#endif
-  teardown_service (&sh);
-  GNUNET_free_non_null (sh.handlers);
-  GNUNET_SPEEDUP_stop_ ();
-  GNUNET_CONFIGURATION_destroy (cfg);
-  GNUNET_free_non_null (logfile);
-  GNUNET_free_non_null (loglev);
-  GNUNET_free (cfg_filename);
-  GNUNET_free_non_null (opt_cfg_filename);
-
-  return err ? GNUNET_SYSERR : sh.ret;
-}
-
-
-/**
- * Suspend accepting connections from the listen socket temporarily.
- * Resume activity using #GNUNET_SERVICE_resume.
- *
- * @param sh service to stop accepting connections.
- */
-void
-GNUNET_SERVICE_suspend (struct GNUNET_SERVICE_Handle *sh)
-{
-  struct ServiceListenContext *slc;
-
-  for (slc = sh->slc_head; NULL != slc; slc = slc->next)
-  {
-    if (NULL != slc->listen_task)
-    {
-      GNUNET_SCHEDULER_cancel (slc->listen_task);
-      slc->listen_task = NULL;
-    }
-  }
-}
-
-
-/**
- * Task run when we are ready to transmit data to the
- * client.
- *
- * @param cls the `struct GNUNET_SERVICE_Client *` to send to
- */
-static void
-do_send (void *cls)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-  ssize_t ret;
-  size_t left;
-  const char *buf;
-
-  client->send_task = NULL;
-  buf = (const char *) client->msg;
-  left = ntohs (client->msg->size) - client->msg_pos;
-  ret = GNUNET_NETWORK_socket_send (client->sock,
-                                    &buf[client->msg_pos],
-                                    left);
-  GNUNET_assert (ret <= (ssize_t) left);
-  if (0 == ret)
-  {
-    GNUNET_MQ_inject_error (client->mq,
-                            GNUNET_MQ_ERROR_WRITE);
-    return;
-  }
-  if (-1 == ret)
-  {
-    if ( (EAGAIN == errno) ||
-         (EINTR == errno) )
-    {
-      /* ignore */
-      ret = 0;
-    }
-    else
-    {
-      if (EPIPE != errno)
-        GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
-                             "send");
-      GNUNET_MQ_inject_error (client->mq,
-                              GNUNET_MQ_ERROR_WRITE);
-      return;
-    }
-  }
-  if (0 == client->msg_pos)
-  {
-    GNUNET_MQ_impl_send_in_flight (client->mq);
-  }
-  client->msg_pos += ret;
-  if (left > ret)
-  {
-    GNUNET_assert (NULL == client->drop_task);
-    client->send_task
-      = GNUNET_SCHEDULER_add_write_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                        client->sock,
-                                        &do_send,
-                                        client);
-    return;
-  }
-  GNUNET_MQ_impl_send_continue (client->mq);
-}
-
-
-/**
- * Signature of functions implementing the sending functionality of a
- * message queue.
- *
- * @param mq the message queue
- * @param msg the message to send
- * @param impl_state our `struct GNUNET_SERVICE_Client *`
- */
-static void
-service_mq_send (struct GNUNET_MQ_Handle *mq,
-                 const struct GNUNET_MessageHeader *msg,
-                 void *impl_state)
-{
-  struct GNUNET_SERVICE_Client *client = impl_state;
-
-  if (NULL != client->drop_task)
-    return; /* we're going down right now, do not try to send */
-  GNUNET_assert (NULL == client->send_task);
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "Sending message of type %u and size %u to client\n",
-       ntohs (msg->type), ntohs (msg->size));
-
-  client->msg = msg;
-  client->msg_pos = 0;
-  client->send_task
-    = GNUNET_SCHEDULER_add_write_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                      client->sock,
-                                      &do_send,
-                                      client);
-}
-
-
-/**
- * Implementation function that cancels the currently sent message.
- *
- * @param mq message queue
- * @param impl_state state specific to the implementation
- */
-static void
-service_mq_cancel (struct GNUNET_MQ_Handle *mq,
-                   void *impl_state)
-{
-  struct GNUNET_SERVICE_Client *client = impl_state;
-
-  GNUNET_assert (0 == client->msg_pos);
-  client->msg = NULL;
-  GNUNET_SCHEDULER_cancel (client->send_task);
-  client->send_task = NULL;
-}
-
-
-/**
- * Generic error handler, called with the appropriate
- * error code and the same closure specified at the creation of
- * the message queue.
- * Not every message queue implementation supports an error handler.
- *
- * @param cls closure with our `struct GNUNET_SERVICE_Client`
- * @param error error code
- */
-static void
-service_mq_error_handler (void *cls,
-                          enum GNUNET_MQ_Error error)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-  struct GNUNET_SERVICE_Handle *sh = client->sh;
-
-  if ( (GNUNET_MQ_ERROR_NO_MATCH == error) &&
-       (GNUNET_NO == sh->require_found) )
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "No handler for message of type %u found\n",
-                (unsigned int) client->warn_type);
-    GNUNET_SERVICE_client_continue (client);
-    return; /* ignore error */
-  }
-  GNUNET_SERVICE_client_drop (client);
-}
-
-
-/**
- * Task run to warn about missing calls to #GNUNET_SERVICE_client_continue().
- *
- * @param cls our `struct GNUNET_SERVICE_Client *` to process more requests from
- */
-static void
-warn_no_client_continue (void *cls)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-
-  GNUNET_break (0 != client->warn_type); /* type should never be 0 here, as we don't use 0 */
-  client->warn_task
-    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                    &warn_no_client_continue,
-                                    client);
-  LOG (GNUNET_ERROR_TYPE_WARNING,
-       _("Processing code for message of type %u did not call `GNUNET_SERVICE_client_continue' after %s\n"),
-       (unsigned int) client->warn_type,
-       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (client->warn_start),
-                                               GNUNET_YES));
-}
-
-
-/**
- * Functions with this signature are called whenever a
- * complete message is received by the tokenizer for a client.
- *
- * Do not call #GNUNET_MST_destroy() from within
- * the scope of this callback.
- *
- * @param cls closure with the `struct GNUNET_SERVICE_Client *`
- * @param message the actual message
- * @return #GNUNET_OK on success, #GNUNET_SYSERR if the client was dropped
- */
-static int
-service_client_mst_cb (void *cls,
-                       const struct GNUNET_MessageHeader *message)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-
-  LOG (GNUNET_ERROR_TYPE_INFO,
-       "Received message of type %u and size %u from client\n",
-       ntohs (message->type), ntohs (message->size));
-
-  GNUNET_assert (GNUNET_NO == client->needs_continue);
-  client->needs_continue = GNUNET_YES;
-  client->warn_type = ntohs (message->type);
-  client->warn_start = GNUNET_TIME_absolute_get ();
-  GNUNET_assert (NULL == client->warn_task);
-  client->warn_task
-    = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
-                                    &warn_no_client_continue,
-                                    client);
-  GNUNET_MQ_inject_message (client->mq,
-                            message);
-  if (NULL != client->drop_task)
-    return GNUNET_SYSERR;
-  return GNUNET_OK;
-}
-
-
-/**
- * A client sent us data. Receive and process it.  If we are done,
- * reschedule this task.
- *
- * @param cls the `struct GNUNET_SERVICE_Client` that sent us data.
- */
-static void
-service_client_recv (void *cls)
-{
-  struct GNUNET_SERVICE_Client *client = cls;
-  int ret;
-
-  client->recv_task = NULL;
-  ret = GNUNET_MST_read (client->mst,
-                         client->sock,
-                         GNUNET_NO,
-                         GNUNET_YES);
-  if (GNUNET_SYSERR == ret)
-  {
-    /* client closed connection (or IO error) */
-    if (NULL == client->drop_task)
-    {
-      GNUNET_assert (GNUNET_NO == client->needs_continue);
-      GNUNET_SERVICE_client_drop (client);
-    }
-    return;
-  }
-  if (GNUNET_NO == ret)
-    return; /* more messages in buffer, wait for application
-               to be done processing */
-  GNUNET_assert (GNUNET_OK == ret);
-  if (GNUNET_YES == client->needs_continue)
-    return;
-  if (NULL != client->recv_task)
-    return;
-  /* MST needs more data, re-schedule read job */
-  client->recv_task
-    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                     client->sock,
-                                     &service_client_recv,
-                                     client);
-}
-
-
-/**
- * We have successfully accepted a connection from a client.  Now
- * setup the client (with the scheduler) and tell the application.
- *
- * @param sh service that accepted the client
- * @param sock socket associated with the client
- */
-static void
-start_client (struct GNUNET_SERVICE_Handle *sh,
-              struct GNUNET_NETWORK_Handle *csock)
-{
-  struct GNUNET_SERVICE_Client *client;
-
-  client = GNUNET_new (struct GNUNET_SERVICE_Client);
-  GNUNET_CONTAINER_DLL_insert (sh->clients_head,
-                               sh->clients_tail,
-                               client);
-  client->sh = sh;
-  client->sock = csock;
-  client->mq = GNUNET_MQ_queue_for_callbacks (&service_mq_send,
-                                              NULL,
-                                              &service_mq_cancel,
-                                              client,
-                                              sh->handlers,
-                                              &service_mq_error_handler,
-                                              client);
-  client->mst = GNUNET_MST_create (&service_client_mst_cb,
-                                   client);
-  if (NULL != sh->connect_cb)
-    client->user_context = sh->connect_cb (sh->cb_cls,
-                                           client,
-                                           client->mq);
-  GNUNET_MQ_set_handlers_closure (client->mq,
-                                  client->user_context);
-  client->recv_task
-    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                     client->sock,
-                                     &service_client_recv,
-                                     client);
-}
-
-
-/**
- * Check if the given IP address is in the list of IP addresses.
- *
- * @param list a list of networks
- * @param add the IP to check (in network byte order)
- * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
- */
-static int
-check_ipv4_listed (const struct GNUNET_STRINGS_IPv4NetworkPolicy *list,
-                   const struct in_addr *add)
-{
-  unsigned int i;
-
-  if (NULL == list)
-    return GNUNET_NO;
-  i = 0;
-  while ( (0 != list[i].network.s_addr) ||
-          (0 != list[i].netmask.s_addr) )
-  {
-    if ((add->s_addr & list[i].netmask.s_addr) ==
-        (list[i].network.s_addr & list[i].netmask.s_addr))
-      return GNUNET_YES;
-    i++;
-  }
-  return GNUNET_NO;
-}
-
-
-/**
- * Check if the given IP address is in the list of IP addresses.
- *
- * @param list a list of networks
- * @param ip the IP to check (in network byte order)
- * @return #GNUNET_NO if the IP is not in the list, #GNUNET_YES if it it is
- */
-static int
-check_ipv6_listed (const struct GNUNET_STRINGS_IPv6NetworkPolicy *list,
-                   const struct in6_addr *ip)
-{
-  unsigned int i;
-  unsigned int j;
-  struct in6_addr zero;
-
-  if (NULL == list)
-    return GNUNET_NO;
-  memset (&zero,
-          0,
-          sizeof (struct in6_addr));
-  i = 0;
-NEXT:
-  while (0 != memcmp (&zero,
-                      &list[i].network,
-                      sizeof (struct in6_addr)))
-  {
-    for (j = 0; j < sizeof (struct in6_addr) / sizeof (int); j++)
-      if (((((int *) ip)[j] & ((int *) &list[i].netmask)[j])) !=
-          (((int *) &list[i].network)[j] & ((int *) &list[i].netmask)[j]))
-      {
-        i++;
-        goto NEXT;
-      }
-    return GNUNET_YES;
-  }
-  return GNUNET_NO;
-}
-
-
-/**
- * We have a client. Accept the incoming socket(s) (and reschedule
- * the listen task).
- *
- * @param cls the `struct ServiceListenContext` of the ready listen socket
- */
-static void
-accept_client (void *cls)
-{
-  struct ServiceListenContext *slc = cls;
-  struct GNUNET_SERVICE_Handle *sh = slc->sh;
-
-  slc->listen_task = NULL;
-  while (1)
-  {
-    struct GNUNET_NETWORK_Handle *sock;
-    const struct sockaddr_in *v4;
-    const struct sockaddr_in6 *v6;
-    struct sockaddr_storage sa;
-    socklen_t addrlen;
-    int ok;
-
-    addrlen = sizeof (sa);
-    sock = GNUNET_NETWORK_socket_accept (slc->listen_socket,
-                                         (struct sockaddr *) &sa,
-                                         &addrlen);
-    if (NULL == sock)
-      break;
-    switch (sa.ss_family)
-    {
-    case AF_INET:
-      GNUNET_assert (addrlen == sizeof (struct sockaddr_in));
-      v4 = (const struct sockaddr_in *) &sa;
-      ok = ( ( (NULL == sh->v4_allowed) ||
-               (check_ipv4_listed (sh->v4_allowed,
-                                   &v4->sin_addr))) &&
-             ( (NULL == sh->v4_denied) ||
-               (! check_ipv4_listed (sh->v4_denied,
-                                     &v4->sin_addr)) ) );
-      break;
-    case AF_INET6:
-      GNUNET_assert (addrlen == sizeof (struct sockaddr_in6));
-      v6 = (const struct sockaddr_in6 *) &sa;
-      ok = ( ( (NULL == sh->v6_allowed) ||
-               (check_ipv6_listed (sh->v6_allowed,
-                                   &v6->sin6_addr))) &&
-             ( (NULL == sh->v6_denied) ||
-               (! check_ipv6_listed (sh->v6_denied,
-                                     &v6->sin6_addr)) ) );
-      break;
-#ifndef WINDOWS
-    case AF_UNIX:
-      ok = GNUNET_OK;            /* controlled using file-system ACL now */
-      break;
-#endif
-    default:
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           _("Unknown address family %d\n"),
-           sa.ss_family);
-      return;
-    }
-    if (! ok)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Service rejected incoming connection from %s due to policy.\n",
-           GNUNET_a2s ((const struct sockaddr *) &sa,
-                       addrlen));
-      GNUNET_break (GNUNET_OK ==
-                    GNUNET_NETWORK_socket_close (sock));
-      continue;
-    }
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Service accepted incoming connection from %s.\n",
-         GNUNET_a2s ((const struct sockaddr *) &sa,
-                     addrlen));
-    start_client (slc->sh,
-                  sock);
-  }
-  slc->listen_task
-    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                     slc->listen_socket,
-                                     &accept_client,
-                                     slc);
-}
-
-
-/**
- * Resume accepting connections from the listen socket.
- *
- * @param sh service to resume accepting connections.
- */
-void
-GNUNET_SERVICE_resume (struct GNUNET_SERVICE_Handle *sh)
-{
-  struct ServiceListenContext *slc;
-
-  for (slc = sh->slc_head; NULL != slc; slc = slc->next)
-  {
-    GNUNET_assert (NULL == slc->listen_task);
-    slc->listen_task
-      = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                       slc->listen_socket,
-                                       &accept_client,
-                                       slc);
-  }
-}
-
-
-/**
- * Task run to resume receiving data from the client after
- * the client called #GNUNET_SERVICE_client_continue().
- *
- * @param cls our `struct GNUNET_SERVICE_Client`
- */
-static void
-resume_client_receive (void *cls)
-{
-  struct GNUNET_SERVICE_Client *c = cls;
-  int ret;
-
-  c->recv_task = NULL;
-  /* first, check if there is still something in the buffer */
-  ret = GNUNET_MST_next (c->mst,
-                         GNUNET_YES);
-  if (GNUNET_SYSERR == ret)
-  {
-    if (NULL != c->drop_task)
-      GNUNET_SERVICE_client_drop (c);
-    return;
-  }
-  if (GNUNET_NO == ret)
-    return; /* done processing, wait for more later */
-  GNUNET_assert (GNUNET_OK == ret);
-  if (GNUNET_YES == c->needs_continue)
-    return; /* #GNUNET_MST_next() did give a message to the client */
-  /* need to receive more data from the network first */
-  if (NULL != c->recv_task)
-    return;
-  c->recv_task
-    = GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                     c->sock,
-                                     &service_client_recv,
-                                     c);
-}
-
-
-/**
- * Continue receiving further messages from the given client.
- * Must be called after each message received.
- *
- * @param c the client to continue receiving from
- */
-void
-GNUNET_SERVICE_client_continue (struct GNUNET_SERVICE_Client *c)
-{
-  GNUNET_assert (GNUNET_YES == c->needs_continue);
-  GNUNET_assert (NULL == c->recv_task);
-  c->needs_continue = GNUNET_NO;
-  if (NULL != c->warn_task)
-  {
-    GNUNET_SCHEDULER_cancel (c->warn_task);
-    c->warn_task = NULL;
-  }
-  c->recv_task
-    = GNUNET_SCHEDULER_add_now (&resume_client_receive,
-                                c);
-}
-
-
-/**
- * Disable the warning the server issues if a message is not
- * acknowledged in a timely fashion.  Use this call if a client is
- * intentionally delayed for a while.  Only applies to the current
- * message.
- *
- * @param c client for which to disable the warning
- */
-void
-GNUNET_SERVICE_client_disable_continue_warning (struct GNUNET_SERVICE_Client *c)
-{
-  GNUNET_break (NULL != c->warn_task);
-  if (NULL != c->warn_task)
-  {
-    GNUNET_SCHEDULER_cancel (c->warn_task);
-    c->warn_task = NULL;
-  }
-}
-
-
-/**
- * Asynchronously finish dropping the client.
- *
- * @param cls the `struct GNUNET_SERVICE_Client`.
- */
-static void
-finish_client_drop (void *cls)
-{
-  struct GNUNET_SERVICE_Client *c = cls;
-  struct GNUNET_SERVICE_Handle *sh = c->sh;
-
-  c->drop_task = NULL;
-  GNUNET_assert (NULL == c->send_task);
-  GNUNET_assert (NULL == c->recv_task);
-  GNUNET_assert (NULL == c->warn_task);
-  GNUNET_MST_destroy (c->mst);
-  GNUNET_MQ_destroy (c->mq);
-  if (GNUNET_NO == c->persist)
-  {
-    GNUNET_break (GNUNET_OK ==
-                  GNUNET_NETWORK_socket_close (c->sock));
-  }
-  else
-  {
-    GNUNET_NETWORK_socket_free_memory_only_ (c->sock);
-  }
-  GNUNET_free (c);
-  if ( (GNUNET_YES == sh->got_shutdown) &&
-       (GNUNET_NO == have_non_monitor_clients (sh)) )
-    GNUNET_SERVICE_shutdown (sh);
-}
-
-
-/**
- * Ask the server to disconnect from the given client.  This is the
- * same as returning #GNUNET_SYSERR within the check procedure when
- * handling a message, wexcept that it allows dropping of a client even
- * when not handling a message from that client.  The `disconnect_cb`
- * will be called on @a c even if the application closes the connection
- * using this function.
- *
- * @param c client to disconnect now
- */
-void
-GNUNET_SERVICE_client_drop (struct GNUNET_SERVICE_Client *c)
-{
-  struct GNUNET_SERVICE_Handle *sh = c->sh;
-
-  if (NULL != c->drop_task)
-  {
-    /* asked to drop twice! */
-    GNUNET_assert (0);
-    return;
-  }
-  GNUNET_CONTAINER_DLL_remove (sh->clients_head,
-                               sh->clients_tail,
-                               c);
-  if (NULL != sh->disconnect_cb)
-    sh->disconnect_cb (sh->cb_cls,
-                       c,
-                       c->user_context);
-  if (NULL != c->warn_task)
-  {
-    GNUNET_SCHEDULER_cancel (c->warn_task);
-    c->warn_task = NULL;
-  }
-  if (NULL != c->recv_task)
-  {
-    GNUNET_SCHEDULER_cancel (c->recv_task);
-    c->recv_task = NULL;
-  }
-  if (NULL != c->send_task)
-  {
-    GNUNET_SCHEDULER_cancel (c->send_task);
-    c->send_task = NULL;
-  }
-  c->drop_task = GNUNET_SCHEDULER_add_now (&finish_client_drop,
-                                           c);
-}
-
-
-/**
- * Explicitly stops the service.
- *
- * @param sh server to shutdown
- */
-void
-GNUNET_SERVICE_shutdown (struct GNUNET_SERVICE_Handle *sh)
-{
-  struct GNUNET_SERVICE_Client *client;
-
-  GNUNET_SERVICE_suspend (sh);
-  sh->got_shutdown = GNUNET_NO;
-  while (NULL != (client = sh->clients_head))
-    GNUNET_SERVICE_client_drop (client);
-}
-
-
-/**
- * Set the 'monitor' flag on this client.  Clients which have been
- * marked as 'monitors' won't prevent the server from shutting down
- * once #GNUNET_SERVICE_stop_listening() has been invoked.  The idea is
- * that for "normal" clients we likely want to allow them to process
- * their requests; however, monitor-clients are likely to 'never'
- * disconnect during shutdown and thus will not be considered when
- * determining if the server should continue to exist after
- * shutdown has been triggered.
- *
- * @param c client to mark as a monitor
- */
-void
-GNUNET_SERVICE_client_mark_monitor (struct GNUNET_SERVICE_Client *c)
-{
-  c->is_monitor = GNUNET_YES;
-  if ( (GNUNET_YES == c->sh->got_shutdown) &&
-       (GNUNET_NO == have_non_monitor_clients (c->sh)) )
-    GNUNET_SERVICE_shutdown (c->sh);
-}
-
-
-/**
- * Set the persist option on this client.  Indicates that the
- * underlying socket or fd should never really be closed.  Used for
- * indicating process death.
- *
- * @param c client to persist the socket (never to be closed)
- */
-void
-GNUNET_SERVICE_client_persist (struct GNUNET_SERVICE_Client *c)
-{
-  c->persist = GNUNET_YES;
-}
-
-
-/**
- * Obtain the message queue of @a c.  Convenience function.
- *
- * @param c the client to continue receiving from
- * @return the message queue of @a c
- */
-struct GNUNET_MQ_Handle *
-GNUNET_SERVICE_client_get_mq (struct GNUNET_SERVICE_Client *c)
-{
-  return c->mq;
-}
-
-
-/* end of service_new.c */
diff --git a/src/util/socks.c b/src/util/socks.c
index 37e60e3e4..85548fd79 100644
--- a/src/util/socks.c
+++ b/src/util/socks.c
@@ -372,15 +372,18 @@ transmit_ready (void *cls,
     return 0;
   }
 
-  GNUNET_assert (1024 >= size && size > 0);
-  GNUNET_assert (SOCKS5_step_done > ih->step && ih->step >= 0);
+  GNUNET_assert ( (1024 >= size) && (size > 0) );
+  GNUNET_assert ( (SOCKS5_step_done > ih->step) && (ih->step >= 0) );
   unsigned char * b = ih->outstep[ih->step];
   unsigned char * e = ih->outstep[ih->step+1];
   GNUNET_assert (e <= &ih->outbuf[1024]);
-  unsigned l = e - b;
-  GNUNET_assert (size >= l && l >= 0);
-  GNUNET_memcpy(buf, b, l);
-  register_reciever (ih, register_reciever_wants(ih));
+  unsigned int l = e - b;
+  GNUNET_assert (size >= l);
+  GNUNET_memcpy (buf,
+                 b,
+                 l);
+  register_reciever (ih,
+                     register_reciever_wants (ih));
   return l;
 }
 
@@ -566,17 +569,25 @@ GNUNET_SOCKS_check_service (const char *service_name,
  */
 struct GNUNET_CONNECTION_Handle *
 GNUNET_SOCKS_do_connect (const char *service_name,
-                          const struct GNUNET_CONFIGURATION_Handle *cfg)
+                         const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct GNUNET_SOCKS_Handshake *ih;
   struct GNUNET_CONNECTION_Handle *socks5; /* *proxied */
-  char *host0,*host1,*user,*pass;
-  unsigned long long port0,port1;
-
-  if (GNUNET_YES != GNUNET_SOCKS_check_service (service_name, cfg))
+  char *host0;
+  char *host1;
+  char *user;
+  char *pass;
+  unsigned long long port0;
+  unsigned long long port1;
+
+  if (GNUNET_YES !=
+      GNUNET_SOCKS_check_service (service_name, cfg))
     return NULL;
   if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_get_value_number (cfg, service_name, "SOCKSPORT", &port0))
+      GNUNET_CONFIGURATION_get_value_number (cfg,
+                                             service_name,
+                                             "SOCKSPORT",
+                                             &port0))
     port0 = 9050;
   /* A typical Tor client should usually try port 9150 for the TBB too, but
    * GNUnet can probably assume a system Tor installation. */
@@ -588,16 +599,23 @@ GNUNET_SOCKS_do_connect (const char *service_name,
          service_name);
     return NULL;
   }
-  if ((GNUNET_OK !=
-       GNUNET_CONFIGURATION_get_value_number (cfg, service_name, "PORT", &port1))
-      || (port1 > 65535) || (port1 <= 0) ||
+  if ( (GNUNET_OK !=
+        GNUNET_CONFIGURATION_get_value_number (cfg,
+                                               service_name,
+                                               "PORT",
+                                               &port1)) ||
+       (port1 > 65535) ||
+       (port1 <= 0) ||
        (GNUNET_OK !=
-        GNUNET_CONFIGURATION_get_value_string (cfg, service_name, "HOSTNAME", &host1)))
+        GNUNET_CONFIGURATION_get_value_string (cfg,
+                                               service_name,
+                                               "HOSTNAME",
+                                               &host1)))
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
-         _
-         ("Attempting to proxy service `%s' to invalid port %d or hostname `%s'.\n"),
-         service_name,port1,host1);
+         _("Attempting to proxy service `%s' to invalid port %d or hostname.\n"),
+         service_name,
+         port1);
     return NULL;
   }
   /* Appeared to still work after host0 corrupted, so either test case is broken, or
@@ -605,20 +623,32 @@ GNUNET_SOCKS_do_connect (const char *service_name,
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_string (cfg, service_name, "SOCKSHOST", &host0))
     host0 = NULL;
-  socks5 = GNUNET_CONNECTION_create_from_connect (cfg, (host0 != NULL)? host0:"127.0.0.1", port0);
+  socks5 = GNUNET_CONNECTION_create_from_connect (cfg,
+                                                  (host0 != NULL)
+                                                  ? host0
+                                                  :"127.0.0.1",
+                                                  port0);
   GNUNET_free_non_null (host0);
 
   /* Sets to NULL if they do not exist */
-  (void) GNUNET_CONFIGURATION_get_value_string (cfg, service_name, "SOCKSUSER", &user);
-  (void) GNUNET_CONFIGURATION_get_value_string (cfg, service_name, "SOCKSPASS", &pass);
+  (void) GNUNET_CONFIGURATION_get_value_string (cfg,
+                                                service_name,
+                                                "SOCKSUSER",
+                                                &user);
+  (void) GNUNET_CONFIGURATION_get_value_string (cfg,
+                                                service_name,
+                                                "SOCKSPASS",
+                                                &pass);
   ih = GNUNET_SOCKS_init_handshake(user,pass);
-  if (NULL != user) GNUNET_free (user);
-  if (NULL != pass) GNUNET_free (pass);
+  GNUNET_free_non_null (user);
+  GNUNET_free_non_null (pass);
 
-  GNUNET_SOCKS_set_handshake_destination (ih,host1,port1);
+  GNUNET_SOCKS_set_handshake_destination (ih,
+                                          host1,
+                                          port1);
   GNUNET_free (host1);
-
-  return GNUNET_SOCKS_run_handshake(ih,socks5);
+  return GNUNET_SOCKS_run_handshake (ih,
+                                     socks5);
 }
 
 /* socks.c */
diff --git a/src/util/test_client.c b/src/util/test_client.c
index f60e5b7f7..527b400b0 100644
--- a/src/util/test_client.c
+++ b/src/util/test_client.c
@@ -179,7 +179,7 @@ main (int argc,
     test_argv[2] = "test_client_unix.conf";
   global_ret = 1;
   if (0 !=
-      GNUNET_SERVICE_ruN_ (3,
+      GNUNET_SERVICE_run_ (3,
                            test_argv,
                            "test_client",
                            GNUNET_SERVICE_OPTION_NONE,
diff --git a/src/util/test_common_allocation.c b/src/util/test_common_allocation.c
index 4ef98b629..4d1b6fe7d 100644
--- a/src/util/test_common_allocation.c
+++ b/src/util/test_common_allocation.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2001, 2002, 2003, 2005, 2006 GNUnet e.V.
+     Copyright (C) 2001, 2002, 2003, 2005, 2006, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -25,6 +25,7 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
+
 static int
 check ()
 {
@@ -95,47 +96,57 @@ check ()
   if (ptrs[0] != NULL)
     return 9;
 
-        /* GNUNET_new_array_2d tests */
-        a2 = GNUNET_new_array_2d (17, 22, unsigned int);
-        for (i = 0; i < 17; i++)
-        {
-                for (j = 0; j < 22; j++)
-                {
-                        if (0 != a2[i][j])
-                                return 10;
-                        a2[i][j] = i * 100 + j;
-                }
-        }
-        free (a2);
-
-        /* GNUNET_new_array_3d tests */
-        a3 = GNUNET_new_array_3d (2, 3, 4, char);
-        for (i = 0; i < 2; i++)
-        {
-                for (j = 0; j < 3; j++)
-                {
-                        for (k = 0; k < 4; k++)
-                        {
-                                if (0 != a3[i][j][k])
-                                        return 11;
-                                a3[i][j][k] = i * 100 + j * 10 + k;
-                        }
-                }
-        }
-        free (a3);
+  /* GNUNET_new_array_2d tests */
+  a2 = GNUNET_new_array_2d (17, 22, unsigned int);
+  for (i = 0; i < 17; i++)
+  {
+    for (j = 0; j < 22; j++)
+    {
+      if (0 != a2[i][j])
+      {
+        GNUNET_free (a2);
+        return 10;
+      }
+      a2[i][j] = i * 100 + j;
+    }
+  }
+  GNUNET_free (a2);
 
+  /* GNUNET_new_array_3d tests */
+  a3 = GNUNET_new_array_3d (2, 3, 4, char);
+  for (i = 0; i < 2; i++)
+  {
+    for (j = 0; j < 3; j++)
+    {
+      for (k = 0; k < 4; k++)
+      {
+        if (0 != a3[i][j][k])
+        {
+          GNUNET_free (a3);
+          return 11;
+        }
+        a3[i][j][k] = i * 100 + j * 10 + k;
+      }
+    }
+  }
+  GNUNET_free (a3);
   return 0;
 }
 
+
 int
 main (int argc, char *argv[])
 {
   int ret;
 
-  GNUNET_log_setup ("test-common-allocation", "WARNING", NULL);
+  GNUNET_log_setup ("test-common-allocation",
+                    "WARNING",
+                    NULL);
   ret = check ();
   if (ret != 0)
-    FPRINTF (stderr, "ERROR %d.\n", ret);
+    FPRINTF (stderr,
+             "ERROR %d.\n",
+             ret);
   return ret;
 }
 
diff --git a/src/util/test_connection.c b/src/util/test_connection.c
deleted file mode 100644
index eaca75c2e..000000000
--- a/src/util/test_connection.c
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection.c
- * @brief tests for connection.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 12435
-
-
-static struct GNUNET_CONNECTION_Handle *csock;
-
-static struct GNUNET_CONNECTION_Handle *asock;
-
-static struct GNUNET_CONNECTION_Handle *lsock;
-
-static size_t sofar;
-
-static struct GNUNET_NETWORK_Handle *ls;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-/**
- * Create and initialize a listen socket for the server.
- *
- * @return -1 on error, otherwise the listen socket
- */
-static struct GNUNET_NETWORK_Handle *
-open_listen_socket ()
-{
-  const static int on = 1;
-  struct sockaddr_in sa;
-  struct GNUNET_NETWORK_Handle *desc;
-
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_port = htons (PORT);
-  sa.sin_family = AF_INET;
-  desc = GNUNET_NETWORK_socket_create (AF_INET, SOCK_STREAM, 0);
-  GNUNET_assert (desc != NULL);
-  if (GNUNET_NETWORK_socket_setsockopt
-      (desc, SOL_SOCKET, SO_REUSEADDR, &on, sizeof (on)) != GNUNET_OK)
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "setsockopt");
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_NETWORK_socket_bind (desc, (const struct sockaddr *) &sa,
-                                             sizeof (sa)));
-  GNUNET_NETWORK_socket_listen (desc, 5);
-  return desc;
-}
-
-static void
-receive_check (void *cls, const void *buf, size_t available,
-               const struct sockaddr *addr, socklen_t addrlen, int errCode)
-{
-  int *ok = cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Receive validates incoming data\n");
-  GNUNET_assert (buf != NULL);  /* no timeout */
-  if (0 == memcmp (&"Hello World"[sofar], buf, available))
-    sofar += available;
-  if (sofar < 12)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Receive needs more data\n");
-    GNUNET_CONNECTION_receive (asock, 1024,
-                               GNUNET_TIME_relative_multiply
-                               (GNUNET_TIME_UNIT_SECONDS, 5), &receive_check,
-                               cls);
-  }
-  else
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Receive closes accepted socket\n");
-    *ok = 0;
-    GNUNET_CONNECTION_destroy (asock);
-    GNUNET_CONNECTION_destroy (csock);
-  }
-}
-
-
-static void
-run_accept (void *cls)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test accepts connection\n");
-  asock = GNUNET_CONNECTION_create_from_accept (NULL, NULL, ls);
-  GNUNET_assert (asock != NULL);
-  GNUNET_assert (GNUNET_YES == GNUNET_CONNECTION_check (asock));
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test destroys listen socket\n");
-  GNUNET_CONNECTION_destroy (lsock);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Test asks to receive on accepted socket\n");
-  GNUNET_CONNECTION_receive (asock, 1024,
-                             GNUNET_TIME_relative_multiply
-                             (GNUNET_TIME_UNIT_SECONDS, 5), &receive_check,
-                             cls);
-}
-
-
-static size_t
-make_hello (void *cls, size_t size, void *buf)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Test prepares to transmit on connect socket\n");
-  GNUNET_assert (size >= 12);
-  strcpy ((char *) buf, "Hello World");
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test destroys client socket\n");
-  return 12;
-}
-
-
-static void
-task (void *cls)
-{
-  ls = open_listen_socket ();
-  lsock = GNUNET_CONNECTION_create_from_existing (ls);
-  GNUNET_assert (lsock != NULL);
-  csock = GNUNET_CONNECTION_create_from_connect (cfg, "localhost", PORT);
-  GNUNET_assert (csock != NULL);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test asks for write notification\n");
-  GNUNET_assert (NULL !=
-                 GNUNET_CONNECTION_notify_transmit_ready (csock, 12,
-                                                          GNUNET_TIME_UNIT_SECONDS,
-                                                          &make_hello, NULL));
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test prepares to accept\n");
-  GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL, ls, &run_accept,
-                                 cls);
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ok;
-
-  GNUNET_log_setup ("test_connection",
-                    "WARNING",
-                    NULL);
-
-  ok = 1;
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  GNUNET_SCHEDULER_run (&task, &ok);
-  GNUNET_CONFIGURATION_destroy (cfg);
-  return ok;
-}
-
-/* end of test_connection.c */
diff --git a/src/util/test_connection_addressing.c b/src/util/test_connection_addressing.c
deleted file mode 100644
index a6345b10a..000000000
--- a/src/util/test_connection_addressing.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection_addressing.c
- * @brief tests for connection.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-
-#define PORT 12435
-
-
-static struct GNUNET_CONNECTION_Handle *csock;
-
-static struct GNUNET_CONNECTION_Handle *asock;
-
-static struct GNUNET_CONNECTION_Handle *lsock;
-
-static size_t sofar;
-
-static struct GNUNET_NETWORK_Handle *ls;
-
-
-
-/**
- * Create and initialize a listen socket for the server.
- *
- * @return NULL on error, otherwise the listen socket
- */
-static struct GNUNET_NETWORK_Handle *
-open_listen_socket ()
-{
-  const static int on = 1;
-  struct sockaddr_in sa;
-  struct GNUNET_NETWORK_Handle *desc;
-
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  desc = GNUNET_NETWORK_socket_create (AF_INET, SOCK_STREAM, 0);
-  GNUNET_assert (desc != 0);
-  if (GNUNET_NETWORK_socket_setsockopt
-      (desc, SOL_SOCKET, SO_REUSEADDR, &on, sizeof (on)) != GNUNET_OK)
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "setsockopt");
-  if (GNUNET_OK !=
-      GNUNET_NETWORK_socket_bind (desc, (const struct sockaddr *) &sa,
-                                  sizeof (sa)))
-  {
-    GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                         "bind");
-    GNUNET_assert (0);
-  }
-  GNUNET_NETWORK_socket_listen (desc, 5);
-  return desc;
-}
-
-
-static void
-receive_check (void *cls, const void *buf, size_t available,
-               const struct sockaddr *addr, socklen_t addrlen, int errCode)
-{
-  int *ok = cls;
-
-  GNUNET_assert (buf != NULL);  /* no timeout */
-  if (0 == memcmp (&"Hello World"[sofar], buf, available))
-    sofar += available;
-  if (sofar < 12)
-  {
-    GNUNET_CONNECTION_receive (asock, 1024,
-                               GNUNET_TIME_relative_multiply
-                               (GNUNET_TIME_UNIT_SECONDS, 5), &receive_check,
-                               cls);
-  }
-  else
-  {
-    *ok = 0;
-    GNUNET_CONNECTION_destroy (csock);
-    GNUNET_CONNECTION_destroy (asock);
-  }
-}
-
-
-static void
-run_accept (void *cls)
-{
-  void *addr;
-  size_t alen;
-  struct sockaddr_in *v4;
-  struct sockaddr_in expect;
-
-  asock = GNUNET_CONNECTION_create_from_accept (NULL, NULL, ls);
-  GNUNET_assert (asock != NULL);
-  GNUNET_assert (GNUNET_YES == GNUNET_CONNECTION_check (asock));
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_CONNECTION_get_address (asock, &addr, &alen));
-  GNUNET_assert (alen == sizeof (struct sockaddr_in));
-  v4 = addr;
-  memset (&expect, 0, sizeof (expect));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  expect.sin_len = sizeof (expect);
-#endif
-  expect.sin_family = AF_INET;
-  expect.sin_port = v4->sin_port;
-  expect.sin_addr.s_addr = htonl (INADDR_LOOPBACK);
-  GNUNET_assert (0 == memcmp (&expect, v4, alen));
-  GNUNET_free (addr);
-  GNUNET_CONNECTION_destroy (lsock);
-  GNUNET_CONNECTION_receive (asock, 1024,
-                             GNUNET_TIME_relative_multiply
-                             (GNUNET_TIME_UNIT_SECONDS, 5), &receive_check,
-                             cls);
-}
-
-static size_t
-make_hello (void *cls, size_t size, void *buf)
-{
-  GNUNET_assert (size >= 12);
-  strcpy ((char *) buf, "Hello World");
-  return 12;
-}
-
-
-static void
-task (void *cls)
-{
-  struct sockaddr_in v4;
-
-  ls = open_listen_socket ();
-  lsock = GNUNET_CONNECTION_create_from_existing (ls);
-  GNUNET_assert (lsock != NULL);
-
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  v4.sin_len = sizeof (v4);
-#endif
-  v4.sin_family = AF_INET;
-  v4.sin_port = htons (PORT);
-  v4.sin_addr.s_addr = htonl (INADDR_LOOPBACK);
-  csock =
-      GNUNET_CONNECTION_create_from_sockaddr (AF_INET,
-                                              (const struct sockaddr *) &v4,
-                                              sizeof (v4));
-  GNUNET_assert (csock != NULL);
-  GNUNET_assert (NULL !=
-                 GNUNET_CONNECTION_notify_transmit_ready (csock, 12,
-                                                          GNUNET_TIME_UNIT_SECONDS,
-                                                          &make_hello, NULL));
-  GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL, ls, &run_accept,
-                                 cls);
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ok;
-
-  GNUNET_log_setup ("test_connection_addressing",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  GNUNET_SCHEDULER_run (&task, &ok);
-  return ok;
-}
-
-/* end of test_connection_addressing.c */
diff --git a/src/util/test_connection_receive_cancel.c b/src/util/test_connection_receive_cancel.c
deleted file mode 100644
index 9c0ab699e..000000000
--- a/src/util/test_connection_receive_cancel.c
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection_receive_cancel.c
- * @brief tests for connection.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 12435
-
-
-static struct GNUNET_CONNECTION_Handle *csock;
-
-static struct GNUNET_CONNECTION_Handle *asock;
-
-static struct GNUNET_CONNECTION_Handle *lsock;
-
-static struct GNUNET_NETWORK_Handle *ls;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-
-/**
- * Create and initialize a listen socket for the server.
- *
- * @return NULL on error, otherwise the listen socket
- */
-static struct GNUNET_NETWORK_Handle *
-open_listen_socket ()
-{
-  const static int on = 1;
-  struct sockaddr_in sa;
-  struct GNUNET_NETWORK_Handle *desc;
-
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  desc = GNUNET_NETWORK_socket_create (AF_INET, SOCK_STREAM, 0);
-  GNUNET_assert (desc != NULL);
-  if (GNUNET_NETWORK_socket_setsockopt
-      (desc, SOL_SOCKET, SO_REUSEADDR, &on, sizeof (on)) != GNUNET_OK)
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK,
-                "setsockopt");
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_NETWORK_socket_bind (desc, (const struct sockaddr *) &sa,
-                                             sizeof (sa)));
-  GNUNET_NETWORK_socket_listen (desc, 5);
-  return desc;
-}
-
-
-static void
-dead_receive (void *cls,
-              const void *buf,
-              size_t available,
-              const struct sockaddr *addr,
-              socklen_t addrlen,
-              int errCode)
-{
-  GNUNET_assert (0);
-}
-
-
-static void
-run_accept_cancel (void *cls)
-{
-  asock = GNUNET_CONNECTION_create_from_accept (NULL, NULL, ls);
-  GNUNET_assert (asock != NULL);
-  GNUNET_assert (GNUNET_YES == GNUNET_CONNECTION_check (asock));
-  GNUNET_CONNECTION_destroy (lsock);
-  GNUNET_CONNECTION_receive (asock, 1024,
-                             GNUNET_TIME_relative_multiply
-                             (GNUNET_TIME_UNIT_SECONDS, 5),
-                             &dead_receive, cls);
-}
-
-
-static void
-receive_cancel_task (void *cls)
-{
-  int *ok = cls;
-
-  GNUNET_CONNECTION_receive_cancel (asock);
-  GNUNET_CONNECTION_destroy (csock);
-  GNUNET_CONNECTION_destroy (asock);
-  *ok = 0;
-}
-
-
-static void
-task_receive_cancel (void *cls)
-{
-  ls = open_listen_socket ();
-  lsock = GNUNET_CONNECTION_create_from_existing (ls);
-  GNUNET_assert (lsock != NULL);
-  csock = GNUNET_CONNECTION_create_from_connect (cfg, "localhost", PORT);
-  GNUNET_assert (csock != NULL);
-  GNUNET_SCHEDULER_add_read_net (GNUNET_TIME_UNIT_FOREVER_REL,
-                                 ls,
-                                 &run_accept_cancel,
-                                 cls);
-  GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
-                                &receive_cancel_task,
-                                cls);
-}
-
-
-/**
- * Main method, starts scheduler with task_timeout.
- */
-static int
-check_receive_cancel ()
-{
-  int ok;
-
-  ok = 1;
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_string (cfg,
-                                         "resolver",
-                                         "HOSTNAME",
-                                         "localhost");
-  GNUNET_SCHEDULER_run (&task_receive_cancel, &ok);
-  GNUNET_CONFIGURATION_destroy (cfg);
-  return ok;
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ret = 0;
-
-  GNUNET_log_setup ("test_connection_receive_cancel", "WARNING", NULL);
-  ret += check_receive_cancel ();
-
-  return ret;
-}
-
-/* end of test_connection_receive_cancel.c */
diff --git a/src/util/test_connection_timeout.c b/src/util/test_connection_timeout.c
deleted file mode 100644
index e78cec669..000000000
--- a/src/util/test_connection_timeout.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection_timeout.c
- * @brief tests for connection.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 12435
-
-static struct GNUNET_CONNECTION_Handle *csock;
-
-static struct GNUNET_CONNECTION_Handle *lsock;
-
-static struct GNUNET_NETWORK_Handle *ls;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-
-/**
- * Create and initialize a listen socket for the server.
- *
- * @return NULL on error, otherwise the listen socket
- */
-static struct GNUNET_NETWORK_Handle *
-open_listen_socket ()
-{
-  const static int on = 1;
-  struct sockaddr_in sa;
-  struct GNUNET_NETWORK_Handle *desc;
-
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  desc = GNUNET_NETWORK_socket_create (AF_INET, SOCK_STREAM, 0);
-  GNUNET_assert (desc != NULL);
-  if (GNUNET_NETWORK_socket_setsockopt
-      (desc, SOL_SOCKET, SO_REUSEADDR, &on, sizeof (on)) != GNUNET_OK)
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, "setsockopt");
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_NETWORK_socket_bind (desc, (const struct sockaddr *) &sa,
-                                             sizeof (sa)));
-  GNUNET_NETWORK_socket_listen (desc, 5);
-  return desc;
-}
-
-
-static size_t
-send_kilo (void *cls, size_t size, void *buf)
-{
-  int *ok = cls;
-
-  if (size == 0)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got the desired timeout!\n");
-    GNUNET_assert (buf == NULL);
-    *ok = 0;
-    GNUNET_CONNECTION_destroy (lsock);
-    GNUNET_CONNECTION_destroy (csock);
-    return 0;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Sending kilo to fill buffer.\n");
-  GNUNET_assert (size >= 1024);
-  memset (buf, 42, 1024);
-
-  GNUNET_assert (NULL !=
-                 GNUNET_CONNECTION_notify_transmit_ready (csock, 1024,
-                                                          GNUNET_TIME_UNIT_SECONDS,
-                                                          &send_kilo, cls));
-  return 1024;
-}
-
-
-static void
-task_timeout (void *cls)
-{
-
-  ls = open_listen_socket ();
-  lsock = GNUNET_CONNECTION_create_from_existing (ls);
-  GNUNET_assert (lsock != NULL);
-  csock = GNUNET_CONNECTION_create_from_connect (cfg, "localhost", PORT);
-  GNUNET_assert (csock != NULL);
-  GNUNET_assert (NULL !=
-                 GNUNET_CONNECTION_notify_transmit_ready (csock, 1024,
-                                                          GNUNET_TIME_UNIT_SECONDS,
-                                                          &send_kilo, cls));
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ok;
-
-  GNUNET_log_setup ("test_connection_timeout",
-                    "WARNING",
-                    NULL);
-
-  ok = 1;
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  GNUNET_SCHEDULER_run (&task_timeout, &ok);
-  GNUNET_CONFIGURATION_destroy (cfg);
-  return ok;
-}
-
-/* end of test_connection_timeout.c */
diff --git a/src/util/test_connection_timeout_no_connect.c b/src/util/test_connection_timeout_no_connect.c
deleted file mode 100644
index ebcd4b71e..000000000
--- a/src/util/test_connection_timeout_no_connect.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection_timeout_no_connect.c
- * @brief tests for connection.c, doing timeout which connect failure
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 13425
-
-static struct GNUNET_CONNECTION_Handle *csock;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-static size_t
-handle_timeout (void *cls, size_t size, void *buf)
-{
-  int *ok = cls;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received timeout signal.\n");
-  GNUNET_assert (size == 0);
-  GNUNET_assert (buf == NULL);
-  *ok = 0;
-  return 0;
-}
-
-
-static void
-task_timeout (void *cls)
-{
-  csock = GNUNET_CONNECTION_create_from_connect (cfg, "localhost", PORT);
-  GNUNET_assert (csock != NULL);
-  GNUNET_assert (NULL !=
-                 GNUNET_CONNECTION_notify_transmit_ready (csock, 1024,
-                                                          GNUNET_TIME_UNIT_SECONDS,
-                                                          &handle_timeout,
-                                                          cls));
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ok;
-
-  GNUNET_log_setup ("test_connection_timeout_no_connect",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  GNUNET_SCHEDULER_run (&task_timeout, &ok);
-  GNUNET_CONFIGURATION_destroy (cfg);
-  return ok;
-}
-
-/* end of test_connection_timeout_no_connect.c */
diff --git a/src/util/test_connection_transmit_cancel.c b/src/util/test_connection_transmit_cancel.c
deleted file mode 100644
index 9ef0720ed..000000000
--- a/src/util/test_connection_transmit_cancel.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_connection_transmit_cancel.c
- * @brief tests for connection.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 12435
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-
-static size_t
-not_run (void *cls, size_t size, void *buf)
-{
-  GNUNET_assert (0);
-  return 0;
-}
-
-
-static void
-task_transmit_cancel (void *cls)
-{
-  int *ok = cls;
-  struct GNUNET_CONNECTION_TransmitHandle *th;
-  struct GNUNET_CONNECTION_Handle *csock;
-
-  csock = GNUNET_CONNECTION_create_from_connect (cfg, "localhost", PORT);
-  GNUNET_assert (csock != NULL);
-  th = GNUNET_CONNECTION_notify_transmit_ready (csock, 12,
-                                                GNUNET_TIME_UNIT_MINUTES,
-                                                &not_run, cls);
-  GNUNET_assert (NULL != th);
-  GNUNET_CONNECTION_notify_transmit_ready_cancel (th);
-  GNUNET_CONNECTION_destroy (csock);
-  *ok = 0;
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ok;
-
-  GNUNET_log_setup ("test_connection_transmit_cancel",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  GNUNET_SCHEDULER_run (&task_transmit_cancel, &ok);
-  GNUNET_CONFIGURATION_destroy (cfg);
-  return ok;
-}
-
-/* end of test_connection_transmit_cancel.c */
diff --git a/src/util/test_getopt.c b/src/util/test_getopt.c
index 8e578640d..13cedd7f5 100644
--- a/src/util/test_getopt.c
+++ b/src/util/test_getopt.c
@@ -136,13 +136,16 @@ testLogOpts ()
     GNUNET_GETOPT_OPTION_END
   };
 
-  if (5 != GNUNET_GETOPT_run ("test_getopt", logoptionlist, 5, myargv))
+  if (5 != GNUNET_GETOPT_run ("test_getopt",
+                              logoptionlist,
+                              5, myargv))
   {
     GNUNET_break (0);
     return 1;
   }
-  GNUNET_assert (fn != NULL);
-  if ((0 != strcmp (level, "WARNING")) || (0 != strcmp (fn, "filename")))
+  GNUNET_assert (NULL != fn);
+  if ( (0 != strcmp (level, "WARNING")) ||
+       (NULL == strstr (fn, "/filename")) )
   {
     GNUNET_break (0);
     GNUNET_free (level);
@@ -170,21 +173,35 @@ testFlagNum ()
   unsigned long long lnum = 0;
 
   const struct GNUNET_GETOPT_CommandLineOption logoptionlist[] = {
-    {'f', "--flag", NULL, "helptext", 0, &GNUNET_GETOPT_set_one,
-     (void *) &flag},
-    {'n', "--num", "ARG", "helptext", 1, &GNUNET_GETOPT_set_uint,
-     (void *) &num},
-    {'N', "--lnum", "ARG", "helptext", 1, &GNUNET_GETOPT_set_ulong,
-     (void *) &lnum},
+    GNUNET_GETOPT_OPTION_SET_ONE ('f',
+                                  "--flag",
+                                  "helptext",
+                                  &flag),
+    GNUNET_GETOPT_OPTION_SET_UINT ('n',
+                                   "--num",
+                                   "ARG",
+                                   "helptext",
+                                   &num),
+    GNUNET_GETOPT_OPTION_SET_ULONG ('N',
+                                    "--lnum",
+                                    "ARG",
+                                    "helptext",
+                                    &lnum),
     GNUNET_GETOPT_OPTION_END
   };
 
-  if (6 != GNUNET_GETOPT_run ("test_getopt", logoptionlist, 6, myargv))
+  if (6 !=
+      GNUNET_GETOPT_run ("test_getopt",
+                         logoptionlist,
+                         6,
+                         myargv))
   {
     GNUNET_break (0);
     return 1;
   }
-  if ((1 != flag) || (42 != num) || (42 != lnum))
+  if ( (1 != flag) ||
+       (42 != num) ||
+       (42 != lnum))
   {
     GNUNET_break (0);
     return 1;
@@ -198,7 +215,9 @@ main (int argc, char *argv[])
 {
   int errCnt = 0;
 
-  GNUNET_log_setup ("test_getopt", "WARNING", NULL);
+  GNUNET_log_setup ("test_getopt",
+                    "WARNING",
+                    NULL);
   /* suppress output from -h, -v options */
 #ifndef MINGW
   GNUNET_break (0 == CLOSE (1));
diff --git a/src/util/test_program.c b/src/util/test_program.c
index 669cee7bd..d206952af 100644
--- a/src/util/test_program.c
+++ b/src/util/test_program.c
@@ -24,37 +24,19 @@
 #include "platform.h"
 #include "gnunet_util_lib.h"
 
-static int setme1, setme2;
-
-static struct GNUNET_GETOPT_CommandLineOption options1[] = {
-  {'n', "name", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme1},
-  GNUNET_GETOPT_OPTION_END
-};
-
-static struct GNUNET_GETOPT_CommandLineOption options2[] = {
-  {'n', "name", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme1},
-  {'N', "number", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme2},
-  GNUNET_GETOPT_OPTION_END
-};
-
-static struct GNUNET_GETOPT_CommandLineOption options3[] = {
-  {'N', "number", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme1},
-  {'n', "name", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme2},
-  GNUNET_GETOPT_OPTION_END
-};
-
-static struct GNUNET_GETOPT_CommandLineOption options4[] = {
-  {'n', "name", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme1},
-  {'n', "number", NULL, "description", 0, &GNUNET_GETOPT_set_one, &setme2},
-  GNUNET_GETOPT_OPTION_END
-};
+
+static int setme1;
+
+static int setme2;
+
 
 /**
  * Main function that will be run.
  */
-
 static void
-runner (void *cls, char *const *args, const char *cfgfile,
+runner (void *cls,
+        char *const *args,
+        const char *cfgfile,
         const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   int *ok = cls;
@@ -62,21 +44,16 @@ runner (void *cls, char *const *args, const char *cfgfile,
   GNUNET_assert (setme1 == 1);
   GNUNET_assert (0 == strcmp (args[0], "extra"));
   GNUNET_assert (args[1] == NULL);
-  GNUNET_assert (0 == strcmp (cfgfile, "test_program_data.conf"));
-
+  GNUNET_assert (NULL != strstr (cfgfile, "/test_program_data.conf"));
   *ok = 0;
 }
 
-/**
- * Main method, starts scheduler with task1,
- * checks that "ok" is correct at the end.
- */
-static int
-check ()
+
+int
+main (int argc, char *argv[])
 {
   int ok = 1;
-
-  char *const argv[] = {
+  char *const argvx[] = {
     "test_program",
     "-c",
     "test_program_data.conf",
@@ -86,33 +63,75 @@ check ()
     "extra",
     NULL
   };
+  struct GNUNET_GETOPT_CommandLineOption options1[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "name",
+                                  "description",
+                                  &setme1),
+    GNUNET_GETOPT_OPTION_END
+  };
+  struct GNUNET_GETOPT_CommandLineOption options2[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "name",
+                                  "description",
+                                  &setme1),
+    GNUNET_GETOPT_OPTION_SET_ONE ('N',
+                                  "number",
+                                  "description",
+                                  &setme2),
+    GNUNET_GETOPT_OPTION_END
+  };
+  struct GNUNET_GETOPT_CommandLineOption options3[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('N',
+                                  "number",
+                                  "description",
+                                  &setme1),
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "name",
+                                  "description",
+                                  &setme2),
+    GNUNET_GETOPT_OPTION_END
+  };
+  struct GNUNET_GETOPT_CommandLineOption options4[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "name",
+                                  "description",
+                                  &setme1),
+    GNUNET_GETOPT_OPTION_SET_ONE ('n',
+                                  "name",
+                                  "description",
+                                  &setme2),
+    GNUNET_GETOPT_OPTION_END
+  };
 
+
+  GNUNET_log_setup ("test_program",
+                    "WARNING",
+                    NULL);
   GNUNET_assert (GNUNET_OK ==
-                 GNUNET_PROGRAM_run (7, argv, "test_program", "A test",
-                                     options1, &runner, &ok));
+                 GNUNET_PROGRAM_run (7, argvx,
+                                     "test_program",
+                                     "A test",
+                                     options1,
+                                     &runner, &ok));
 
   GNUNET_assert (GNUNET_OK ==
-                 GNUNET_PROGRAM_run (7, argv, "test_program", "A test",
-                                     options2, &runner, &ok));
+                 GNUNET_PROGRAM_run (7, argvx,
+                                     "test_program", "A test",
+                                     options2,
+                                     &runner, &ok));
   GNUNET_assert (GNUNET_OK ==
-                 GNUNET_PROGRAM_run (7, argv, "test_program", "A test",
-                                     options3, &runner, &ok));
+                 GNUNET_PROGRAM_run (7, argvx,
+                                     "test_program", "A test",
+                                     options3,
+                                     &runner, &ok));
   GNUNET_assert (GNUNET_OK ==
-                 GNUNET_PROGRAM_run (7, argv, "test_program", "A test",
-                                     options4, &runner, &ok));
+                 GNUNET_PROGRAM_run (7, argvx,
+                                     "test_program", "A test",
+                                     options4,
+                                     &runner, &ok));
 
   return ok;
 }
 
-int
-main (int argc, char *argv[])
-{
-  int ret = 0;
-
-  GNUNET_log_setup ("test_program", "WARNING", NULL);
-  ret += check ();
-
-  return ret;
-}
-
 /* end of test_program.c */
diff --git a/src/util/test_server.c b/src/util/test_server.c
deleted file mode 100644
index 8003adbf4..000000000
--- a/src/util/test_server.c
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2010, 2014, 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_server.c
- * @brief tests for server.c
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-/**
- * TCP port to use for the server.
- */
-#define PORT 12435
-
-/**
- * Timeout to use for operations.
- */
-#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 2)
-
-/**
- * Test message type.
- */
-#define MY_TYPE 128
-
-/**
- * Test message type.
- */
-#define MY_TYPE2 129
-
-/**
- * Handle for the server.
- */
-static struct GNUNET_SERVER_Handle *server;
-
-/**
- * Handle for the client.
- */
-static struct GNUNET_MQ_Handle *mq;
-
-/**
- * Handle of the server for the client.
- */
-static struct GNUNET_SERVER_Client *argclient;
-
-/**
- * Our configuration.
- */
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-/**
- * Number indiciating in which phase of the test we are.
- */
-static int ok;
-
-
-/**
- * Final task invoked to clean up.
- *
- * @param cls NULL
- */
-static void
-finish_up (void *cls)
-{
-  GNUNET_assert (7 == ok);
-  ok = 0;
-  GNUNET_SERVER_destroy (server);
-  GNUNET_MQ_destroy (mq);
-  GNUNET_CONFIGURATION_destroy (cfg);
-}
-
-
-/**
- * The server has received the second message, initiate clean up.
- *
- * @param cls NULL
- * @param client client we got the message from
- * @param message the message
- */
-static void
-recv_fin_cb (void *cls,
-             struct GNUNET_SERVER_Client *client,
-             const struct GNUNET_MessageHeader *message)
-{
-  GNUNET_assert (6 == ok);
-  ok = 7;
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-  GNUNET_SCHEDULER_add_now (&finish_up, NULL);
-}
-
-
-/**
- * We have received the reply from the server, check that we are at
- * the right stage and queue the next message to the server.  Cleans
- * up #argclient.
- *
- * @param cls NULL
- * @param msg message we got from the server
- */
-static void
-handle_reply (void *cls,
-              const struct GNUNET_MessageHeader *msg)
-{
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *m;
-
-  GNUNET_assert (4 == ok);
-  ok = 6;
-  env = GNUNET_MQ_msg (m,
-                       MY_TYPE2);
-  GNUNET_MQ_send (mq,
-                  env);
-}
-
-
-/**
- * Send a reply of type #MY_TYPE from the server to the client.
- * Checks that we are in the right phase and transmits the
- * reply.  Cleans up #argclient state.
- *
- * @param cls NULL
- * @param size number of bytes we are allowed to send
- * @param buf where to copy the reply
- * @return number of bytes written to @a buf
- */
-static size_t
-reply_msg (void *cls,
-           size_t size,
-           void *buf)
-{
-  struct GNUNET_MessageHeader msg;
-
-  GNUNET_assert (3 == ok);
-  ok = 4;
-  GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader));
-  msg.type = htons (MY_TYPE);
-  msg.size = htons (sizeof (struct GNUNET_MessageHeader));
-  GNUNET_memcpy (buf, &msg, sizeof (struct GNUNET_MessageHeader));
-  GNUNET_assert (NULL != argclient);
-  GNUNET_SERVER_receive_done (argclient, GNUNET_OK);
-  GNUNET_SERVER_client_drop (argclient);
-  argclient = NULL;
-  return sizeof (struct GNUNET_MessageHeader);
-}
-
-
-/**
- * Function called whenever the server receives a message of
- * type #MY_TYPE.  Checks that we are at the stage where
- * we expect the first message, then sends a reply.  Stores
- * the handle to the client in #argclient.
- *
- * @param cls NULL
- * @param client client that sent the message
- * @param message the message we received
- */
-static void
-recv_cb (void *cls,
-         struct GNUNET_SERVER_Client *client,
-         const struct GNUNET_MessageHeader *message)
-{
-  GNUNET_assert (2 == ok);
-  ok = 3;
-  argclient = client;
-  GNUNET_SERVER_client_keep (argclient);
-  GNUNET_assert (sizeof (struct GNUNET_MessageHeader) == ntohs (message->size));
-  GNUNET_assert (MY_TYPE == ntohs (message->type));
-  GNUNET_assert (NULL !=
-                 GNUNET_SERVER_notify_transmit_ready (client,
-                                                      ntohs (message->size),
-                                                      TIMEOUT,
-                                                      &reply_msg,
-                                                      NULL));
-}
-
-
-/**
- * Message handlers for the server.
- */
-static struct GNUNET_SERVER_MessageHandler handlers[] = {
-  {&recv_cb, NULL, MY_TYPE, sizeof (struct GNUNET_MessageHeader)},
-  {&recv_fin_cb, NULL, MY_TYPE2, sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-/**
- * Generic error handler, called with the appropriate error code and
- * the same closure specified at the creation of the message queue.
- * Not every message queue implementation supports an error handler.
- *
- * @param cls closure with the `struct GNUNET_STATISTICS_Handle *`
- * @param error error code
- */
-static void
-mq_error_handler (void *cls,
-                  enum GNUNET_MQ_Error error)
-{
-  GNUNET_assert (0); /* should never happen */
-}
-
-
-/**
- * First task run by the scheduler.  Initializes the server and
- * a client and asks for a transmission from the client to the
- * server.
- *
- * @param cls NULL
- */
-static void
-task (void *cls)
-{
-  struct sockaddr_in sa;
-  struct sockaddr *sap[2];
-  socklen_t slens[2];
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *msg;
-  struct GNUNET_MQ_MessageHandler chandlers[] = {
-    GNUNET_MQ_hd_fixed_size (reply,
-                             MY_TYPE,
-                             struct GNUNET_MessageHeader,
-                             cls),
-    GNUNET_MQ_handler_end ()
-  };
-
-  sap[0] = (struct sockaddr *) &sa;
-  slens[0] = sizeof (sa);
-  sap[1] = NULL;
-  slens[1] = 0;
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  server = GNUNET_SERVER_create (NULL, NULL,
-                                 sap, slens,
-                                 TIMEOUT, GNUNET_NO);
-  GNUNET_assert (server != NULL);
-  GNUNET_SERVER_add_handlers (server, handlers);
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_number (cfg,
-                                         "test-server",
-                                         "PORT",
-                                         PORT);
-  GNUNET_CONFIGURATION_set_value_string (cfg,
-                                         "test-server",
-                                         "HOSTNAME",
-                                         "localhost");
-  GNUNET_CONFIGURATION_set_value_string (cfg,
-                                         "resolver",
-                                         "HOSTNAME",
-                                         "localhost");
-  mq = GNUNET_CLIENT_connect (cfg,
-                              "test-server",
-                              chandlers,
-                              &mq_error_handler,
-                              NULL);
-  GNUNET_assert (NULL != mq);
-  ok = 2;
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-}
-
-
-/**
- * Runs the test.
- *
- * @param argc length of @a argv
- * @param argv command line arguments (ignored)
- * @return 0 on success, otherwise phase of failure
- */
-int
-main (int argc, char *argv[])
-{
-  GNUNET_log_setup ("test_server",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  GNUNET_SCHEDULER_run (&task, &ok);
-  return ok;
-}
-
-/* end of test_server.c */
diff --git a/src/util/test_server_disconnect.c b/src/util/test_server_disconnect.c
deleted file mode 100644
index c3d003e90..000000000
--- a/src/util/test_server_disconnect.c
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2010, 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_server_disconnect.c
- * @brief tests for server.c,  specifically GNUNET_SERVER_client_disconnect
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-
-#define PORT 12435
-
-#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 250)
-
-#define MY_TYPE 128
-
-static struct GNUNET_SERVER_Handle *server;
-
-static struct GNUNET_MQ_Handle *mq;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-static int ok;
-
-
-static void
-finish_up (void *cls)
-{
-  GNUNET_assert (ok == 5);
-  ok = 0;
-  GNUNET_SERVER_destroy (server);
-  GNUNET_MQ_destroy (mq);
-  GNUNET_CONFIGURATION_destroy (cfg);
-}
-
-
-static void
-notify_disconnect (void *cls,
-                   struct GNUNET_SERVER_Client *clientarg)
-{
-  if (NULL == clientarg)
-    return;
-  GNUNET_assert (ok == 4);
-  ok = 5;
-  GNUNET_SCHEDULER_add_now (&finish_up, NULL);
-}
-
-
-static void
-server_disconnect (void *cls)
-{
-  struct GNUNET_SERVER_Client *argclient = cls;
-
-  GNUNET_assert (ok == 3);
-  ok = 4;
-  GNUNET_SERVER_client_disconnect (argclient);
-  GNUNET_SERVER_client_drop (argclient);
-}
-
-
-static void
-recv_cb (void *cls,
-         struct GNUNET_SERVER_Client *client,
-         const struct GNUNET_MessageHeader *message)
-{
-  GNUNET_assert (ok == 2);
-  ok = 3;
-  GNUNET_SERVER_client_keep (client);
-  GNUNET_SCHEDULER_add_now (&server_disconnect, client);
-  GNUNET_assert (sizeof (struct GNUNET_MessageHeader) == ntohs (message->size));
-  GNUNET_assert (MY_TYPE == ntohs (message->type));
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-static struct GNUNET_SERVER_MessageHandler handlers[] = {
-  {&recv_cb, NULL, MY_TYPE, sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-static void
-task (void *cls)
-{
-  struct sockaddr_in sa;
-  struct sockaddr *sap[2];
-  socklen_t slens[2];
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *msg;
-
-  sap[0] = (struct sockaddr *) &sa;
-  slens[0] = sizeof (sa);
-  sap[1] = NULL;
-  slens[1] = 0;
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  server = GNUNET_SERVER_create (NULL, NULL, sap, slens, TIMEOUT, GNUNET_NO);
-  GNUNET_assert (server != NULL);
-  GNUNET_SERVER_add_handlers (server, handlers);
-  GNUNET_SERVER_disconnect_notify (server, &notify_disconnect, NULL);
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_number (cfg, "test-server", "PORT", PORT);
-  GNUNET_CONFIGURATION_set_value_string (cfg, "test-server", "HOSTNAME",
-                                         "localhost");
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  mq = GNUNET_CLIENT_connect (cfg,
-                              "test-server",
-                              NULL,
-                              NULL,
-                              NULL);
-  GNUNET_assert (NULL != mq);
-  ok = 2;
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-}
-
-
-/**
- * Main method, starts scheduler with task1,
- * checks that "ok" is correct at the end.
- */
-static int
-check ()
-{
-  ok = 1;
-  GNUNET_SCHEDULER_run (&task, &ok);
-  return ok;
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  int ret = 0;
-
-  GNUNET_log_setup ("test_server_disconnect", "WARNING", NULL);
-  ret += check ();
-
-  return ret;
-}
-
-/* end of test_server_disconnect.c */
diff --git a/src/util/test_server_mst_interrupt.c b/src/util/test_server_mst_interrupt.c
deleted file mode 100644
index 3141a75bd..000000000
--- a/src/util/test_server_mst_interrupt.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2010 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_server_mst_interrupt.c
- * @brief test for interrupt message processing in server_mst.c
- */
-#include "platform.h"
-#include "gnunet_protocols.h"
-#include "gnunet_util_lib.h"
-
-static struct GNUNET_SERVER_MessageStreamTokenizer * mst;
-
-
-/* Callback destroying mst with data in buffer */
-static int
-mst_cb (void *cls, void *client,
-        const struct GNUNET_MessageHeader * message)
-{
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MST gave me message, destroying\n");
-  GNUNET_SERVER_mst_destroy (mst);
-  return GNUNET_SYSERR;
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  struct GNUNET_PeerIdentity id;
-  struct GNUNET_MessageHeader msg[2];
-
-  GNUNET_log_setup ("test_server_mst_interrupt", "WARNING", NULL);
-  memset (&id, 0, sizeof (id));
-  msg[0].size = htons (sizeof (msg));
-  msg[0].type = htons (sizeof (GNUNET_MESSAGE_TYPE_DUMMY));
-  mst = GNUNET_SERVER_mst_create(mst_cb, NULL);
-  GNUNET_SERVER_mst_receive (mst, &id,
-                             (const char *) &msg, 2 * sizeof (msg),
-                             GNUNET_NO, GNUNET_NO);
-  /* If we reach this line, it did not crash */
-  return 0;
-}
-
-/* end of test_server_mst_interrupt.c */
diff --git a/src/util/test_server_with_client.c b/src/util/test_server_with_client.c
deleted file mode 100644
index 63bfda00c..000000000
--- a/src/util/test_server_with_client.c
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_server_with_client.c
- * @brief tests for server.c and client.c,
- *       specifically disconnect_notify,
- *       client_get_address and receive_done (resume processing)
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define PORT 22335
-
-#define MY_TYPE 128
-
-
-static struct GNUNET_SERVER_Handle *server;
-
-static struct GNUNET_MQ_Handle *mq;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-static int ok;
-
-
-static void
-send_done (void *cls)
-{
-  struct GNUNET_SERVER_Client *argclient = cls;
-
-  GNUNET_assert (ok == 3);
-  ok++;
-  GNUNET_SERVER_receive_done (argclient, GNUNET_OK);
-}
-
-
-static void
-recv_cb (void *cls,
-         struct GNUNET_SERVER_Client *argclient,
-         const struct GNUNET_MessageHeader *message)
-{
-  void *addr;
-  size_t addrlen;
-  struct sockaddr_in sa;
-  struct sockaddr_in *have;
-
-  GNUNET_assert (GNUNET_OK ==
-                 GNUNET_SERVER_client_get_address (argclient,
-                                                   &addr,
-                                                   &addrlen));
-
-  GNUNET_assert (addrlen == sizeof (struct sockaddr_in));
-  have = addr;
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = have->sin_port;
-  sa.sin_addr.s_addr = htonl (INADDR_LOOPBACK);
-  GNUNET_assert (0 == memcmp (&sa, addr, addrlen));
-  GNUNET_free (addr);
-  switch (ok)
-  {
-  case 2:
-    ok++;
-    GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                  (GNUNET_TIME_UNIT_MILLISECONDS, 50),
-                                  &send_done,
-                                  argclient);
-    break;
-  case 4:
-    ok++;
-    GNUNET_MQ_destroy (mq);
-    GNUNET_SERVER_receive_done (argclient,
-                                GNUNET_OK);
-    break;
-  default:
-    GNUNET_assert (0);
-  }
-
-}
-
-
-static void
-clean_up (void *cls)
-{
-  GNUNET_SERVER_destroy (server);
-  server = NULL;
-  GNUNET_CONFIGURATION_destroy (cfg);
-  cfg = NULL;
-}
-
-
-/**
- * Functions with this signature are called whenever a client
- * is disconnected on the network level.
- *
- * @param cls closure
- * @param client identification of the client
- */
-static void
-notify_disconnect (void *cls,
-                   struct GNUNET_SERVER_Client *client)
-{
-  if (client == NULL)
-    return;
-  GNUNET_assert (ok == 5);
-  ok = 0;
-  GNUNET_SCHEDULER_add_now (&clean_up, NULL);
-}
-
-
-static struct GNUNET_SERVER_MessageHandler handlers[] = {
-  {&recv_cb, NULL, MY_TYPE, sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-static void
-task (void *cls)
-{
-  struct sockaddr_in sa;
-  struct sockaddr *sap[2];
-  socklen_t slens[2];
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *msg;
-
-  sap[0] = (struct sockaddr *) &sa;
-  slens[0] = sizeof (sa);
-  sap[1] = NULL;
-  slens[1] = 0;
-  memset (&sa, 0, sizeof (sa));
-#if HAVE_SOCKADDR_IN_SIN_LEN
-  sa.sin_len = sizeof (sa);
-#endif
-  sa.sin_family = AF_INET;
-  sa.sin_port = htons (PORT);
-  server =
-      GNUNET_SERVER_create (NULL, NULL, sap, slens,
-                            GNUNET_TIME_relative_multiply
-                            (GNUNET_TIME_UNIT_MILLISECONDS, 250), GNUNET_NO);
-  GNUNET_assert (server != NULL);
-  handlers[0].callback_cls = cls;
-  GNUNET_SERVER_add_handlers (server, handlers);
-  GNUNET_SERVER_disconnect_notify (server, &notify_disconnect, cls);
-  cfg = GNUNET_CONFIGURATION_create ();
-  GNUNET_CONFIGURATION_set_value_number (cfg, "test", "PORT", PORT);
-  GNUNET_CONFIGURATION_set_value_string (cfg, "test", "HOSTNAME", "localhost");
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  mq = GNUNET_CLIENT_connect (cfg,
-                              "test",
-                              NULL,
-                              NULL,
-                              NULL);
-  GNUNET_assert (NULL != mq);
-  ok = 2;
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  GNUNET_log_setup ("test_server_with_client",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  GNUNET_SCHEDULER_run (&task, NULL);
-  return ok;
-}
-
-/* end of test_server_with_client.c */
diff --git a/src/util/test_server_with_client_unix.c b/src/util/test_server_with_client_unix.c
deleted file mode 100644
index 8fabbe210..000000000
--- a/src/util/test_server_with_client_unix.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
-     This file is part of GNUnet.
-     Copyright (C) 2009, 2016 GNUnet e.V.
-
-     GNUnet is free software; you can redistribute it and/or modify
-     it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 3, or (at your
-     option) any later version.
-
-     GNUnet is distributed in the hope that it will be useful, but
-     WITHOUT ANY WARRANTY; without even the implied warranty of
-     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-     General Public License for more details.
-
-     You should have received a copy of the GNU General Public License
-     along with GNUnet; see the file COPYING.  If not, write to the
-     Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-     Boston, MA 02110-1301, USA.
-*/
-/**
- * @file util/test_server_with_client_unix.c
- * @brief tests for server.c and client.c,
- *       specifically disconnect_notify,
- *       client_get_address and receive_done (resume processing)
- */
-#include "platform.h"
-#include "gnunet_util_lib.h"
-
-#define MY_TYPE 128
-
-
-static struct GNUNET_SERVER_Handle *server;
-
-static struct GNUNET_MQ_Handle *mq;
-
-static struct GNUNET_CONFIGURATION_Handle *cfg;
-
-static int ok;
-
-
-static void
-send_done (void *cls)
-{
-  struct GNUNET_SERVER_Client *argclient = cls;
-
-  GNUNET_assert (ok == 3);
-  ok++;
-  GNUNET_SERVER_receive_done (argclient, GNUNET_OK);
-}
-
-
-static void
-recv_cb (void *cls,
-         struct GNUNET_SERVER_Client *argclient,
-         const struct GNUNET_MessageHeader *message)
-{
-  switch (ok)
-  {
-  case 2:
-    ok++;
-    (void) GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                         (GNUNET_TIME_UNIT_MILLISECONDS, 50),
-                                         &send_done,
-                                         argclient);
-    break;
-  case 4:
-    ok++;
-    GNUNET_MQ_destroy (mq);
-    GNUNET_SERVER_receive_done (argclient, GNUNET_OK);
-    break;
-  default:
-    GNUNET_assert (0);
-  }
-
-}
-
-
-static void
-clean_up (void *cls)
-{
-  GNUNET_SERVER_destroy (server);
-  server = NULL;
-  GNUNET_CONFIGURATION_destroy (cfg);
-  cfg = NULL;
-}
-
-
-/**
- * Functions with this signature are called whenever a client
- * is disconnected on the network level.
- *
- * @param cls closure
- * @param client identification of the client
- */
-static void
-notify_disconnect (void *cls,
-                   struct GNUNET_SERVER_Client *client)
-{
-  if (client == NULL)
-    return;
-  GNUNET_assert (ok == 5);
-  ok = 0;
-  (void) GNUNET_SCHEDULER_add_now (&clean_up, NULL);
-}
-
-
-static struct GNUNET_SERVER_MessageHandler handlers[] = {
-  {&recv_cb, NULL, MY_TYPE, sizeof (struct GNUNET_MessageHeader)},
-  {NULL, NULL, 0, 0}
-};
-
-
-static void
-task (void *cls)
-{
-  struct sockaddr_un un;
-  const char *unixpath = "/tmp/testsock";
-  struct sockaddr *sap[2];
-  socklen_t slens[2];
-  struct GNUNET_MQ_Envelope *env;
-  struct GNUNET_MessageHeader *msg;
-
-  memset (&un, 0, sizeof (un));
-  un.sun_family = AF_UNIX;
-  strncpy(un.sun_path, unixpath, sizeof (un.sun_path) - 1);
-#if HAVE_SOCKADDR_UN_SUN_LEN
-  un.sun_len = (u_char) sizeof (un);
-#endif
-
-  sap[0] = (struct sockaddr *) &un;
-  slens[0] = sizeof (un);
-  sap[1] = NULL;
-  slens[1] = 0;
-  server =
-      GNUNET_SERVER_create (NULL, NULL, sap, slens,
-                            GNUNET_TIME_relative_multiply
-                            (GNUNET_TIME_UNIT_MILLISECONDS, 250), GNUNET_NO);
-  GNUNET_assert (server != NULL);
-  handlers[0].callback_cls = cls;
-  GNUNET_SERVER_add_handlers (server, handlers);
-  GNUNET_SERVER_disconnect_notify (server, &notify_disconnect, cls);
-  cfg = GNUNET_CONFIGURATION_create ();
-
-  GNUNET_CONFIGURATION_set_value_string (cfg, "test", "UNIXPATH", unixpath);
-  GNUNET_CONFIGURATION_set_value_string (cfg, "resolver", "HOSTNAME",
-                                         "localhost");
-  mq = GNUNET_CLIENT_connect (cfg,
-                              "test",
-                              NULL,
-                              NULL,
-                              NULL);
-  GNUNET_assert (NULL != mq);
-  ok = 2;
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-  env = GNUNET_MQ_msg (msg,
-                       MY_TYPE);
-  GNUNET_MQ_send (mq,
-                  env);
-}
-
-
-int
-main (int argc, char *argv[])
-{
-  GNUNET_log_setup ("test_server_with_client_unix",
-                    "WARNING",
-                    NULL);
-  ok = 1;
-  GNUNET_SCHEDULER_run (&task, NULL);
-  return ok;
-}
-
-/* end of test_server_with_client_unix.c */
diff --git a/src/util/test_service.c b/src/util/test_service.c
index d2136b42f..1567c97ce 100644
--- a/src/util/test_service.c
+++ b/src/util/test_service.c
@@ -148,7 +148,7 @@ check (const char *sname)
               sname);
   global_ret = 1;
   GNUNET_assert (0 ==
-                 GNUNET_SERVICE_ruN_ (3,
+                 GNUNET_SERVICE_run_ (3,
                                       argv,
                                       sname,
                                       GNUNET_SERVICE_OPTION_NONE,
diff --git a/src/util/util.conf b/src/util/util.conf
index ecc94ead0..ceb5fdcbb 100644
--- a/src/util/util.conf
+++ b/src/util/util.conf
@@ -37,9 +37,8 @@ GNUNET_RUNTIME_DIR = ${TMPDIR:-${TMP:-/tmp}}/gnunet-system-runtime/
 GNUNET_USER_RUNTIME_DIR = ${TMPDIR:-${TMP:-/tmp}}/gnunet-${USERHOME:-${USER:-user}}-runtime/
 
 
-# Legacy option...
-# GNUNET_TEST_HOME = ~/.gnunet/
-# GNUNET_TEST_HOME = /var/lib/gnunet/
+# Override for GNUNET_HOME used by test cases.
+# GNUNET_TEST_HOME = /tmp/foo/bar
 
 # DEFAULTCONFIG = /etc/gnunet.conf
 # If 'DEFAULTCONFIG' is not defined, the current
diff --git a/src/vpn/Makefile.am b/src/vpn/Makefile.am
index 417d2eb89..5c16fa349 100644
--- a/src/vpn/Makefile.am
+++ b/src/vpn/Makefile.am
@@ -59,7 +59,7 @@ gnunet_service_vpn_LDADD = \
   $(top_builddir)/src/statistics/libgnunetstatistics.la \
   $(top_builddir)/src/tun/libgnunettun.la \
   $(top_builddir)/src/util/libgnunetutil.la \
-  $(top_builddir)/src/cadet/libgnunetcadetnew.la \
+  $(top_builddir)/src/cadet/libgnunetcadet.la \
   $(top_builddir)/src/regex/libgnunetregex.la \
   $(GN_LIBINTL)
 gnunet_service_vpn_CFLAGS = \
diff --git a/src/vpn/gnunet-helper-vpn-windows.c b/src/vpn/gnunet-helper-vpn-windows.c
index a9596752a..e74a0aa2f 100644
--- a/src/vpn/gnunet-helper-vpn-windows.c
+++ b/src/vpn/gnunet-helper-vpn-windows.c
@@ -77,7 +77,7 @@
 static boolean privilege_testing = FALSE;
 
 /**
- * Maximum size of a GNUnet message (GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ * Maximum size of a GNUnet message (GNUNET_MAX_MESSAGE_SIZE)
  */
 #define MAX_SIZE 65536
 
diff --git a/src/vpn/gnunet-helper-vpn.c b/src/vpn/gnunet-helper-vpn.c
index 02889d65b..4ed4e079e 100644
--- a/src/vpn/gnunet-helper-vpn.c
+++ b/src/vpn/gnunet-helper-vpn.c
@@ -53,7 +53,7 @@
 #define DEBUG GNUNET_NO
 
 /**
- * Maximum size of a GNUnet message (GNUNET_SERVER_MAX_MESSAGE_SIZE)
+ * Maximum size of a GNUnet message (GNUNET_MAX_MESSAGE_SIZE)
  */
 #define MAX_SIZE 65536
 
diff --git a/src/vpn/gnunet-service-vpn.c b/src/vpn/gnunet-service-vpn.c
index aa0ea51a3..d9daaa7e2 100644
--- a/src/vpn/gnunet-service-vpn.c
+++ b/src/vpn/gnunet-service-vpn.c
@@ -1384,7 +1384,7 @@ create_channel (struct ChannelState *ts,
     GNUNET_MQ_handler_end()
   };
 
-  return GNUNET_CADET_channel_creatE (cadet_handle,
+  return GNUNET_CADET_channel_create (cadet_handle,
                                       ts,
                                       target,
                                       port,
@@ -1839,7 +1839,7 @@ route_packet (struct DestinationEntry *destination,
 
       mlen = sizeof (struct GNUNET_EXIT_UdpServiceMessage) +
         payload_length - sizeof (struct GNUNET_TUN_UdpHeader);
-      if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
       {
         GNUNET_break (0);
         return;
@@ -1864,7 +1864,7 @@ route_packet (struct DestinationEntry *destination,
 
       mlen = sizeof (struct GNUNET_EXIT_UdpInternetMessage) +
         alen + payload_length - sizeof (struct GNUNET_TUN_UdpHeader);
-      if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
       {
         GNUNET_break (0);
         return;
@@ -1904,7 +1904,7 @@ route_packet (struct DestinationEntry *destination,
 
         mlen = sizeof (struct GNUNET_EXIT_TcpServiceStartMessage) +
           payload_length - sizeof (struct GNUNET_TUN_TcpHeader);
-        if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+        if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
         {
           GNUNET_break (0);
           return;
@@ -1927,7 +1927,7 @@ route_packet (struct DestinationEntry *destination,
 
         mlen = sizeof (struct GNUNET_EXIT_TcpInternetStartMessage) +
           alen + payload_length - sizeof (struct GNUNET_TUN_TcpHeader);
-        if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+        if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
         {
           GNUNET_break (0);
           return;
@@ -1963,7 +1963,7 @@ route_packet (struct DestinationEntry *destination,
 
       mlen = sizeof (struct GNUNET_EXIT_TcpDataMessage) +
         payload_length - sizeof (struct GNUNET_TUN_TcpHeader);
-      if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
       {
         GNUNET_break (0);
         return;
@@ -2038,7 +2038,7 @@ route_packet (struct DestinationEntry *destination,
       /* update length calculations, as payload_length may have changed */
       mlen = sizeof (struct GNUNET_EXIT_IcmpServiceMessage) +
         alen + payload_length - sizeof (struct GNUNET_TUN_IcmpHeader);
-      if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
       {
         GNUNET_break (0);
         return;
@@ -2061,6 +2061,7 @@ route_packet (struct DestinationEntry *destination,
       void *payload;
       uint8_t new_type;
 
+      new_type = icmp->type;
       /* Perform ICMP protocol-translation (depending on destination AF and source AF)
          and throw away ICMP payload depending on ICMP message type */
       switch (af)
@@ -2111,8 +2112,8 @@ route_packet (struct DestinationEntry *destination,
         switch (icmp->type)
           {
           case GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE:
-            if (destination->details.exit_destination.af == AF_INET6)
-              new_type = GNUNET_TUN_ICMPTYPE6_DESTINATION_UNREACHABLE;
+            if (destination->details.exit_destination.af == AF_INET)
+              new_type = GNUNET_TUN_ICMPTYPE_DESTINATION_UNREACHABLE;
             /* throw away IP-payload, exit will have to make it up anyway */
             payload_length = sizeof (struct GNUNET_TUN_IcmpHeader);
             break;
@@ -2167,7 +2168,7 @@ route_packet (struct DestinationEntry *destination,
       /* update length calculations, as payload_length may have changed */
       mlen = sizeof (struct GNUNET_EXIT_IcmpInternetMessage) +
         alen + payload_length - sizeof (struct GNUNET_TUN_IcmpHeader);
-      if (mlen >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
+      if (mlen >= GNUNET_MAX_MESSAGE_SIZE)
       {
         GNUNET_break (0);
         return;
@@ -2216,12 +2217,10 @@ route_packet (struct DestinationEntry *destination,
  * and forward the packet.
  *
  * @param cls closure, NULL
- * @param client NULL
  * @param message message we got from the client (VPN channel interface)
  */
 static int
 message_token (void *cls,
-               void *client,
                const struct GNUNET_MessageHeader *message)
 {
   const struct GNUNET_TUN_Layer2PacketHeader *tun;
@@ -3067,7 +3066,7 @@ run (void *cls,
   }
   vpn_argv[6] = NULL;
 
-  cadet_handle = GNUNET_CADET_connecT (cfg_);
+  cadet_handle = GNUNET_CADET_connect (cfg_);
     // FIXME never opens ports???
   helper_handle = GNUNET_HELPER_start (GNUNET_NO,
                                        "gnunet-helper-vpn", vpn_argv,
diff --git a/src/vpn/gnunet-vpn.c b/src/vpn/gnunet-vpn.c
index 2e7daf7f7..0adbd5c96 100644
--- a/src/vpn/gnunet-vpn.c
+++ b/src/vpn/gnunet-vpn.c
@@ -78,7 +78,7 @@ static int udp;
 /**
  * Selected level of verbosity.
  */
-static int verbosity;
+static unsigned int verbosity;
 
 /**
  * Global return value.
@@ -286,33 +286,53 @@ run (void *cls,
 int
 main (int argc, char *const *argv)
 {
-  static const struct GNUNET_GETOPT_CommandLineOption options[] = {
-    {'4', "ipv4", NULL,
-     gettext_noop ("request that result should be an IPv4 address"),
-     0, &GNUNET_GETOPT_set_one, &ipv4},
-    {'6', "ipv6", NULL,
-     gettext_noop ("request that result should be an IPv6 address"),
-     0, &GNUNET_GETOPT_set_one, &ipv6},
-    {'d', "duration", "TIME",
-     gettext_noop ("how long should the mapping be valid for new tunnels?"),
-     1, &GNUNET_GETOPT_set_relative_time, &duration},
-    {'i', "ip", "IP",
-     gettext_noop ("destination IP for the tunnel"),
-     1, &GNUNET_GETOPT_set_string, &target_ip},
-    {'p', "peer", "PEERID",
-     gettext_noop ("peer offering the service we would like to access"),
-     1, &GNUNET_GETOPT_set_string, &peer_id},
-    {'s', "service", "NAME",
-     gettext_noop ("name of the service we would like to access"),
-     1, &GNUNET_GETOPT_set_string, &service_name},
-    {'t', "tcp", NULL,
-     gettext_noop ("service is offered via TCP"),
-     0, &GNUNET_GETOPT_set_one, &tcp},
-    {'u', "udp", NULL,
-     gettext_noop ("service is offered via UDP"),
-     0, &GNUNET_GETOPT_set_one, &udp},
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_SET_ONE ('4',
+                                  "ipv4",
+                                  gettext_noop ("request that result should be an IPv4 address"),
+                                  &ipv4),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('6',
+                                  "ipv6",
+                                  gettext_noop ("request that result should be an IPv6 address"),
+                                  &ipv6),
+
+    GNUNET_GETOPT_OPTION_SET_RELATIVE_TIME ('d',
+                                            "duration",
+                                            "TIME",
+                                            gettext_noop ("how long should the mapping be valid for new tunnels?"),
+                                            &duration),
+
+    GNUNET_GETOPT_OPTION_STRING ('i',
+                                 "ip",
+                                 "IP",
+                                 gettext_noop ("destination IP for the tunnel"),
+                                 &target_ip),
+
+    GNUNET_GETOPT_OPTION_STRING ('p',
+                                 "peer",
+                                 "PEERID",
+                                 gettext_noop ("peer offering the service we would like to access"),
+                                 &peer_id),
+
+    GNUNET_GETOPT_OPTION_STRING ('s', 
+                                 "service",
+                                 "NAME",
+                                 gettext_noop ("name of the service we would like to access"),
+                                 &service_name),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('t',
+                                  "tcp",
+                                  gettext_noop ("service is offered via TCP"),
+                                  &tcp),
+
+    GNUNET_GETOPT_OPTION_SET_ONE ('u',
+                                  "udp",
+                                  gettext_noop ("service is offered via UDP"),
+                                  &udp),
 
     GNUNET_GETOPT_OPTION_VERBOSE (&verbosity),
+
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))